{
"measurement": {
"model.layers.0.self_attn": [
{
"accuracy": 0.8894891142845154,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9053465723991394,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9185255169868469,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.946878969669342,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.94715416431427,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9496088624000549,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.956382155418396,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9593806862831116,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9659510850906372,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9675307273864746,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9726564884185791,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.977521538734436,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9746923446655273,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9796300530433655,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845275282859802,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881863594055176,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859658479690552,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908760786056519,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940224885940552,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.0.mlp": [
{
"accuracy": 0.8932353258132935,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.8998433947563171,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9166586399078369,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9224433898925781,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9543788433074951,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9582982063293457,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9667852520942688,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9749492406845093,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9767799973487854,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763534069061279,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9784167408943176,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865257740020752,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865979552268982,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900423884391785,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.990635335445404,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916055202484131,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921948909759521,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.1.self_attn": [
{
"accuracy": 0.8912715911865234,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9012794494628906,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9128483533859253,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9391506314277649,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9406632781028748,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9436624050140381,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9507348537445068,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.954677939414978,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9637992978096008,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9650030136108398,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9721841216087341,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9757533073425293,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9739500284194946,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.977653443813324,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836911559104919,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873796701431274,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855232238769531,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.991749107837677,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941839575767517,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.1.mlp": [
{
"accuracy": 0.9552885293960571,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9585185050964355,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9623382687568665,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9627683162689209,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900801181793213,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908599853515625,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918005466461182,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938802123069763,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942141175270081,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956300854682922,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955335855484009,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997096836566925,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968777894973755,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975553750991821,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977337718009949,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978197813034058,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978839159011841,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.2.self_attn": [
{
"accuracy": 0.9904049038887024,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913141131401062,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923480153083801,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936143159866333,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947449564933777,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948529005050659,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964241981506348,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966509938240051,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969533681869507,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970252513885498,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971974492073059,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973798394203186,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997981607913971,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981233477592468,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987816214561462,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989201426506042,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989352822303772,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99924236536026,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994015097618103,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.2.mlp": [
{
"accuracy": 0.9841413497924805,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845923185348511,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871894121170044,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879907965660095,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921269416809082,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927420616149902,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938306212425232,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959120154380798,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963133335113525,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959918856620789,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965057373046875,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979750514030457,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981465935707092,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988325238227844,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988981485366821,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990718364715576,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994029402732849,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.3.self_attn": [
{
"accuracy": 0.9888184070587158,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894803166389465,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99102383852005,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931026697158813,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937865138053894,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939103722572327,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950639009475708,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952230453491211,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958636164665222,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958844780921936,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968673586845398,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971315860748291,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972087144851685,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974961876869202,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998358964920044,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985336661338806,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985257983207703,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990862607955933,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992576837539673,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.3.mlp": [
{
"accuracy": 0.97846919298172,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9790761470794678,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823307394981384,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833335876464844,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892057180404663,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900665283203125,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.991423487663269,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944076538085938,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949561953544617,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945005178451538,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952155351638794,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972234964370728,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974805116653442,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984057545661926,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984928965568542,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987100958824158,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991909265518188,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.4.self_attn": [
{
"accuracy": 0.9872599244117737,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881659746170044,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897461533546448,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916681051254272,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929458498954773,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931999444961548,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944252967834473,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946857690811157,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954175353050232,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953981637954712,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963065385818481,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966523051261902,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967073798179626,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997066080570221,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978436827659607,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982141256332397,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997992753982544,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985576272010803,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987134337425232,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.4.mlp": [
{
"accuracy": 0.9737688899040222,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9745725989341736,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.978746771812439,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800106287002563,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.986883282661438,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879444241523743,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896479249000549,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.993151068687439,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938204884529114,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933164119720459,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941819906234741,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966112375259399,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969204664230347,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980270862579346,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981489181518555,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984211921691895,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989786744117737,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.5.self_attn": [
{
"accuracy": 0.9839826822280884,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.984940767288208,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872392416000366,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901609420776367,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917516112327576,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919210076332092,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940031170845032,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.994178831577301,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950050711631775,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952114820480347,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959136843681335,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996188223361969,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964354634284973,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967801570892334,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979774951934814,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981449842453003,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982814788818359,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986084699630737,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990546107292175,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.5.mlp": [
{
"accuracy": 0.9688599109649658,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9698026180267334,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9747669100761414,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9762660264968872,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844660758972168,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857087135314941,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877287149429321,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919032454490662,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926770925521851,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920963048934937,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931148290634155,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960050582885742,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963874220848083,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977032542228699,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997843861579895,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981726408004761,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988604187965393,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.6.self_attn": [
{
"accuracy": 0.9823703169822693,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832834601402283,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855865836143494,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891499876976013,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910294413566589,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911953806877136,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934511184692383,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937058091163635,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943456053733826,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945772290229797,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955516457557678,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958195090293884,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960156083106995,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963120818138123,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976993203163147,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977706670761108,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980651140213013,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998261570930481,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989122152328491,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.6.mlp": [
{
"accuracy": 0.9659432172775269,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9669575095176697,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9726029634475708,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9743046164512634,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829614162445068,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843642711639404,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866547584533691,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910926222801208,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919700026512146,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913151860237122,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924582242965698,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956016540527344,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960137605667114,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997440755367279,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997607946395874,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979816675186157,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986942410469055,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.7.self_attn": [
{
"accuracy": 0.9785425066947937,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9795452356338501,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818336367607117,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859970808029175,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891411066055298,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894394278526306,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921316504478455,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992537796497345,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929729700088501,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936599135398865,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945806860923767,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947761297225952,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950302243232727,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956609010696411,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971053600311279,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974300861358643,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975470900535583,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980124235153198,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982966184616089,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.7.mlp": [
{
"accuracy": 0.9625129699707031,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9636680483818054,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9696441888809204,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9714412689208984,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813147187232971,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9828659296035767,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852561950683594,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.990254819393158,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912015199661255,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904699921607971,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917256236076355,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951728582382202,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995614230632782,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971889853477478,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973670244216919,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977574348449707,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985550045967102,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.8.self_attn": [
{
"accuracy": 0.9777736067771912,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9787859916687012,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9811835885047913,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861125946044922,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883999824523926,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887996912002563,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916289448738098,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921508431434631,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.993074357509613,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.993094265460968,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99411541223526,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946538209915161,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948044419288635,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952762722969055,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969350099563599,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971960783004761,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974585175514221,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979026913642883,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983522295951843,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.8.mlp": [
{
"accuracy": 0.9601198434829712,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9613378047943115,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9676705002784729,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9695749282836914,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.980120837688446,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817628264427185,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842973947525024,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896200895309448,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906148910522461,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898636341094971,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911804795265198,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948441982269287,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953071475028992,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969735145568848,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971598982810974,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975656867027283,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983972311019897,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.9.self_attn": [
{
"accuracy": 0.9750723838806152,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9758853316307068,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9781551361083984,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9828199744224548,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876522421836853,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879596829414368,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991326630115509,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917554259300232,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992365837097168,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927216172218323,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938943386077881,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939317107200623,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939725399017334,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948351979255676,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968367218971252,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970070123672485,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974533915519714,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976376891136169,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984552264213562,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.9.mlp": [
{
"accuracy": 0.9579552412033081,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9592897891998291,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9659568071365356,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9679789543151855,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9790125489234924,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9807432293891907,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.983441174030304,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890385270118713,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900522232055664,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892924427986145,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907070994377136,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945684671401978,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950594305992126,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968191385269165,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970068335533142,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974364638328552,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99831223487854,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.10.self_attn": [
{
"accuracy": 0.9725459218025208,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.973706066608429,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763246774673462,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9820147752761841,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859477281570435,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864951372146606,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904601573944092,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912437796592712,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918309450149536,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924712181091309,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932141900062561,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936136603355408,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940944314002991,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943674206733704,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966082572937012,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968569874763489,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997394323348999,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997755765914917,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986502528190613,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.10.mlp": [
{
"accuracy": 0.9563877582550049,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9577850699424744,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9650799036026001,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9673100113868713,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9781781435012817,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800301194190979,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829834699630737,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884916543960571,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896602034568787,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888593554496765,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903437495231628,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943460822105408,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.994858980178833,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966739416122437,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969124794006348,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973956942558289,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982843995094299,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.11.self_attn": [
{
"accuracy": 0.9676545858383179,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9690512418746948,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9723002314567566,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9787096381187439,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832901954650879,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839087724685669,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879084229469299,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.98873370885849,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.989790141582489,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905376434326172,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.991787314414978,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923709034919739,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928150177001953,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933564066886902,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958370327949524,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961215257644653,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966230988502502,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99704909324646,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998001217842102,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.11.mlp": [
{
"accuracy": 0.9548360109329224,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9562963247299194,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9638195633888245,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9661181569099426,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774203300476074,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9793193936347961,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823731780052185,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880467653274536,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892613291740417,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884631037712097,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899834394454956,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.994122326374054,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946388006210327,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996505081653595,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967872500419617,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972870349884033,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981887936592102,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.12.self_attn": [
{
"accuracy": 0.9641972184181213,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9658026695251465,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9693731069564819,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9757680892944336,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9812974333763123,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818170070648193,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866162538528442,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872713685035706,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884929656982422,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890803098678589,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906224608421326,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912115931510925,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914998412132263,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923844933509827,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953430891036987,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952540993690491,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996260941028595,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964399933815002,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984259009361267,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.12.mlp": [
{
"accuracy": 0.9506256580352783,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9523324370384216,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9601558446884155,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9625380039215088,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9752918481826782,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774286150932312,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.980593740940094,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869546294212341,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882526993751526,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873430132865906,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890307784080505,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935060739517212,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940733909606934,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960679411888123,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964035749435425,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996902346611023,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978876113891602,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.13.self_attn": [
{
"accuracy": 0.9635826349258423,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9652984142303467,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9685554504394531,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763405323028564,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813709855079651,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9819429516792297,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879060983657837,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886358976364136,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898498058319092,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.990734875202179,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909859299659729,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991578221321106,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922631978988647,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926198124885559,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955633878707886,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958027005195618,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966704249382019,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968808889389038,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983352422714233,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.13.mlp": [
{
"accuracy": 0.9466208219528198,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9483866095542908,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9569805860519409,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9596666693687439,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9731952548027039,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.975504994392395,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9789977073669434,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857807159423828,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872271418571472,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862931966781616,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881302714347839,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930309653282166,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.993647575378418,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958510398864746,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961581230163574,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967268705368042,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977946877479553,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.14.self_attn": [
{
"accuracy": 0.9582585692405701,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9603875875473022,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9646937847137451,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9735056757926941,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9784139394760132,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792734980583191,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855797290802002,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868150353431702,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878315925598145,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886032938957214,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895564913749695,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902179837226868,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909838438034058,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916537404060364,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994742751121521,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952436685562134,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959452748298645,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965469241142273,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980091452598572,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.14.mlp": [
{
"accuracy": 0.9428834319114685,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9448832273483276,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9539346098899841,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9567317962646484,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9714717268943787,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.973924994468689,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9775730967521667,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.984599232673645,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861978888511658,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.985409140586853,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873042106628418,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924609661102295,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930261373519897,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953211545944214,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957422018051147,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963060021400452,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973406195640564,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.15.self_attn": [
{
"accuracy": 0.9561387896537781,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9576238989830017,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.962173342704773,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9698851704597473,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9769297242164612,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9775469303131104,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.983853280544281,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846444725990295,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.985945463180542,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864028096199036,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878035187721252,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891014695167542,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896658062934875,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905022978782654,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937081336975098,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947909116744995,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994744598865509,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964126348495483,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970421195030212,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.15.mlp": [
{
"accuracy": 0.9386206269264221,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9405787587165833,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9503819346427917,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9534177184104919,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9693414568901062,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9718746542930603,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9758535623550415,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.98380446434021,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.985365092754364,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843513369560242,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.986375629901886,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.992042064666748,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927117228507996,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995280921459198,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956230521202087,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962607622146606,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997502863407135,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.16.self_attn": [
{
"accuracy": 0.9568477272987366,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.958450436592102,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9633188247680664,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9713428020477295,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9779719114303589,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9788332581520081,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854867458343506,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866067171096802,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876929521560669,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880092740058899,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889897704124451,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897148609161377,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904581308364868,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913593530654907,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946664571762085,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995122492313385,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958453178405762,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964489936828613,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981192350387573,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.16.mlp": [
{
"accuracy": 0.9321520328521729,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9343292117118835,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9451967477798462,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9486414790153503,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9660667777061462,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9688974022865295,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9733336567878723,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9819574952125549,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837015867233276,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9826695322990417,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849252700805664,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911638498306274,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919082522392273,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946969151496887,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951261878013611,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958317279815674,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971708059310913,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.17.self_attn": [
{
"accuracy": 0.9547853469848633,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9568300843238831,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9611724615097046,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9706187844276428,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9758130311965942,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9766191840171814,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9828602075576782,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837959408760071,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9858819246292114,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866173267364502,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881386160850525,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888018369674683,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894468188285828,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903734922409058,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934695959091187,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941303730010986,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945934414863586,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955435991287231,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997369110584259,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.17.mlp": [
{
"accuracy": 0.9250733256340027,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9274430274963379,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9394822716712952,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9434992671012878,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.962432861328125,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9655452966690063,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9705089926719666,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9798682332038879,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818645715713501,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9807894825935364,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832614660263062,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901688694953918,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.990951657295227,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940311312675476,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945327043533325,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995324432849884,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967342019081116,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.18.self_attn": [
{
"accuracy": 0.9517724514007568,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9536908864974976,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9587340354919434,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9669291377067566,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9743220806121826,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9754363894462585,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9810150861740112,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9824565052986145,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841495752334595,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852151274681091,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872760772705078,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879360795021057,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886556267738342,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897593855857849,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925381541252136,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944015741348267,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934285283088684,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961943626403809,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968279004096985,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.18.mlp": [
{
"accuracy": 0.9197020530700684,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9222961664199829,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9352496266365051,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9397214651107788,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9597401022911072,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9630803465843201,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9684581160545349,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9782376885414124,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9804045557975769,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9793815016746521,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9820182919502258,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894013404846191,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902021288871765,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934722781181335,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940672516822815,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949170351028442,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963622093200684,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.19.self_attn": [
{
"accuracy": 0.9530653953552246,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9547220468521118,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9598905444145203,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9685211181640625,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9755510687828064,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9765381217002869,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9826700091362,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.984046220779419,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854915738105774,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857551455497742,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881002902984619,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888988137245178,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895703792572021,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902517795562744,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935150146484375,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994567334651947,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946010112762451,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959672093391418,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970009326934814,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.19.mlp": [
{
"accuracy": 0.9167290329933167,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9193627238273621,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.932742714881897,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9373924136161804,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.958160936832428,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9616686105728149,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9672916531562805,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774396419525146,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9796855449676514,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785590767860413,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813224673271179,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889391660690308,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897613525390625,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931368827819824,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937783479690552,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946532249450684,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961373805999756,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.20.self_attn": [
{
"accuracy": 0.9575220346450806,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9592010974884033,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9634310603141785,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9716401696205139,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.977070152759552,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9784945249557495,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831038117408752,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850656390190125,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865359663963318,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873391389846802,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890550374984741,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897933602333069,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903711676597595,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906470775604248,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942129850387573,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945792555809021,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951922297477722,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99586421251297,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977531433105469,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.20.mlp": [
{
"accuracy": 0.9154551029205322,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9179303050041199,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9313066005706787,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9356931447982788,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9576643109321594,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9610872268676758,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9666466116905212,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9775826334953308,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9797390699386597,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9784250259399414,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9811834692955017,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890228509902954,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899696111679077,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934812784194946,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939537644386292,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948409795761108,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965232610702515,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.21.self_attn": [
{
"accuracy": 0.961426854133606,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9629462361335754,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9668432474136353,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.974044144153595,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800149202346802,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9807901978492737,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985523521900177,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865506887435913,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.98797208070755,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882016777992249,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.990192174911499,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908323287963867,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913409948348999,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918447732925415,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944695234298706,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951103329658508,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952545762062073,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963668584823608,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975942969322205,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.21.mlp": [
{
"accuracy": 0.9144929647445679,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9169676899909973,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9301540851593018,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9342687726020813,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.957354724407196,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9607127904891968,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9661285877227783,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.977668285369873,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9797379970550537,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9782916903495789,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9810440540313721,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889766573905945,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899887442588806,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935665726661682,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940083026885986,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948758482933044,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967127442359924,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.22.self_attn": [
{
"accuracy": 0.966583251953125,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9682420492172241,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9720790982246399,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.977394163608551,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9812941551208496,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825407862663269,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852385520935059,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868025779724121,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882306456565857,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886466264724731,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911825656890869,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991694450378418,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.992057740688324,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926564693450928,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950656294822693,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959716796875,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956418871879578,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971456527709961,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997920572757721,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.22.mlp": [
{
"accuracy": 0.9135464429855347,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9159696102142334,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9289076328277588,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9328913688659668,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9567651152610779,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9601582288742065,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9654669761657715,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774272441864014,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.979544460773468,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9779962301254272,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9808012843132019,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888661503791809,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899138808250427,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935734868049622,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939566254615784,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948105216026306,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967333078384399,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.23.self_attn": [
{
"accuracy": 0.9633485078811646,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9651376008987427,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9693878293037415,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.975159764289856,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800577163696289,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9814409017562866,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843818545341492,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862643480300903,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874290823936462,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.987926721572876,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904710054397583,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912065863609314,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915816187858582,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921334385871887,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994899570941925,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955587983131409,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955344200134277,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966023564338684,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979627132415771,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.23.mlp": [
{
"accuracy": 0.9116500020027161,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9141713380813599,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9270010590553284,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9309355020523071,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9557231664657593,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9592617750167847,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9645421504974365,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9769132733345032,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9790858626365662,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774635434150696,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9803583025932312,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885847568511963,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896833300590515,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934002757072449,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938130974769592,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946594834327698,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966598153114319,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.24.self_attn": [
{
"accuracy": 0.9636561274528503,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9650927782058716,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9693738222122192,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9751418232917786,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9806642532348633,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817904829978943,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985598087310791,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870457053184509,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877381920814514,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885004758834839,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907394647598267,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912154674530029,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919183850288391,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921905994415283,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951286911964417,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955956339836121,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958378672599792,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967097043991089,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978165626525879,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.24.mlp": [
{
"accuracy": 0.9101490378379822,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9127071499824524,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9254598617553711,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9293527603149414,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.954993486404419,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9586274027824402,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9638591408729553,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.976587176322937,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9787468314170837,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9771195650100708,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.980056643486023,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884234666824341,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895459413528442,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933286905288696,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937210083007812,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945574998855591,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966253638267517,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.25.self_attn": [
{
"accuracy": 0.9617759585380554,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9632810950279236,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9670854806900024,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9725803732872009,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800207018852234,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9814067482948303,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.984876275062561,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865164756774902,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881464242935181,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885151386260986,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906302690505981,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912736415863037,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918791651725769,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924514889717102,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950621724128723,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954215884208679,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957674145698547,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966079592704773,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980592727661133,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.25.mlp": [
{
"accuracy": 0.9077273607254028,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9104152321815491,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9231128096580505,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9270780086517334,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9537777900695801,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9574374556541443,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9626810550689697,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9758293628692627,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9781060814857483,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9764803051948547,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9794782400131226,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881027936935425,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892368316650391,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.993137538433075,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936234354972839,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944758415222168,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966656565666199,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.26.self_attn": [
{
"accuracy": 0.9630751609802246,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.964641809463501,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9686585664749146,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9750838279724121,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.980658233165741,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9815716743469238,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9858152866363525,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870402216911316,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881935119628906,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884188175201416,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903919100761414,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991142988204956,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915738701820374,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921892881393433,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948408603668213,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956203103065491,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995572566986084,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969626665115356,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980353713035583,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.26.mlp": [
{
"accuracy": 0.9052119851112366,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9079035520553589,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9206897020339966,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9246991276741028,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.952405571937561,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9561890363693237,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9614813923835754,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9751007556915283,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.977427065372467,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9757713675498962,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9788720011711121,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877281785011292,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888946413993835,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.992885410785675,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933323860168457,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941787123680115,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996368944644928,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.27.self_attn": [
{
"accuracy": 0.9617952108383179,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.963173508644104,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.968140721321106,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9747693538665771,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9798893332481384,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9806119799613953,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853315353393555,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862959384918213,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877779483795166,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883599877357483,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900802969932556,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909332394599915,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991199791431427,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923754334449768,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948488473892212,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958356022834778,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956928491592407,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971497654914856,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975937008857727,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.27.mlp": [
{
"accuracy": 0.9024766087532043,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9053509831428528,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9183526039123535,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9224793314933777,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9509400725364685,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9548697471618652,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9603148102760315,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9742295145988464,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9766624569892883,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9750002026557922,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9781992435455322,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873186349868774,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885031580924988,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925901889801025,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930939674377441,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939621090888977,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961930513381958,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.28.self_attn": [
{
"accuracy": 0.9562063813209534,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9582061767578125,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9632686972618103,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9708585143089294,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9776896238327026,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785789251327515,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839298725128174,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850220084190369,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863379597663879,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.986682116985321,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890113472938538,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897298216819763,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904634952545166,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905809760093689,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942981004714966,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948861598968506,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952253103256226,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962886571884155,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976072907447815,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.28.mlp": [
{
"accuracy": 0.8957662582397461,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.8988864421844482,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9132251739501953,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9178145527839661,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9475786685943604,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9517873525619507,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9577836394309998,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.972297728061676,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9749296307563782,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9732857942581177,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9766853451728821,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864236116409302,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876409769058228,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919928312301636,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925824403762817,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935377836227417,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958399534225464,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.29.self_attn": [
{
"accuracy": 0.941927969455719,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9442774057388306,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9523913264274597,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9623342752456665,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9708120822906494,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9716720581054688,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800169467926025,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9810842871665955,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829860925674438,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.983744740486145,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856811761856079,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867197275161743,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878730177879333,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886188507080078,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930695295333862,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935817718505859,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994398832321167,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954073429107666,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976582527160645,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.29.mlp": [
{
"accuracy": 0.8949689269065857,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.8981227874755859,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9127395153045654,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9174245595932007,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9470468759536743,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9512619376182556,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9574273228645325,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9718016982078552,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9744392037391663,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.972875714302063,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9762177467346191,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861536622047424,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870167970657349,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916318655014038,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922813177108765,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932299256324768,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953763484954834,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.30.self_attn": [
{
"accuracy": 0.9375888705253601,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9413835406303406,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9522649645805359,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9617159366607666,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9673649668693542,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9669244885444641,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9744256734848022,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9738823771476746,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9777442812919617,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9783174991607666,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848707914352417,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862667322158813,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865623116493225,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879310131072998,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923503994941711,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993171215057373,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993442952632904,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953433275222778,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961714148521423,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.30.mlp": [
{
"accuracy": 0.883132815361023,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.8863840103149414,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9024791121482849,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9074902534484863,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.940670371055603,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.945241391658783,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.951938271522522,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9670978784561157,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9704249501228333,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9685689210891724,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9723235368728638,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836097955703735,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839215874671936,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890198707580566,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905109405517578,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914782643318176,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934507012367249,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.31.self_attn": [
{
"accuracy": 0.9330780506134033,
"total_bits": 89141248,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9363185167312622,
"total_bits": 91697152,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9464724659919739,
"total_bits": 95234560,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9618628621101379,
"total_bits": 111748096,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9682531356811523,
"total_bits": 132388864,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9687350392341614,
"total_bits": 132455936,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792976975440979,
"total_bits": 169089024,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9798822402954102,
"total_bits": 169221632,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818321466445923,
"total_bits": 170671104,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9824144244194031,
"total_bits": 173039616,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843432307243347,
"total_bits": 174398976,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847859144210815,
"total_bits": 175225856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863212704658508,
"total_bits": 178728960,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865679144859314,
"total_bits": 181067776,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918896555900574,
"total_bits": 219944960,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910823702812195,
"total_bits": 223010816,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932389855384827,
"total_bits": 252975104,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924274682998657,
"total_bits": 265314304,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952145218849182,
"total_bits": 336861184,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.31.mlp": [
{
"accuracy": 0.8724724054336548,
"total_bits": 394740800,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.8760221600532532,
"total_bits": 408503360,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.8928354978561401,
"total_bits": 456551424,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.8979451656341553,
"total_bits": 512257024,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9354392886161804,
"total_bits": 577676384,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9413371682167053,
"total_bits": 592151552,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9482553601264954,
"total_bits": 636733536,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9655044078826904,
"total_bits": 728020576,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.969146728515625,
"total_bits": 738755584,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9666789174079895,
"total_bits": 750822496,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.971264123916626,
"total_bits": 765297664,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827379584312439,
"total_bits": 923968608,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841575026512146,
"total_bits": 938443776,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.989135205745697,
"total_bits": 1068803168,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901261329650879,
"total_bits": 1109664000,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911836981773376,
"total_bits": 1209278720,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936026930809021,
"total_bits": 1414799616,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.norm.norm": null,
"lm_head.linear": null
},
"last_module_idx": 66
}