File size: 819 Bytes
40caabd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 |
merge_method: ties # Use TIES for merging multiple models
base_model: Qwen/Qwen2.5-1.5B-Instruct # Base model for the merge
dtype: bfloat16 # Data type for the merged model
models:
- model: Qwen/Qwen2.5-1.5B-Instruct # Base model
parameters:
weight: 0.5 # Weight for the base model
- model: Qwen/Qwen2.5-Math-1.5B-Instruct # Math-focused model
parameters:
density: 0.6 # Retain 60% of significant parameters
weight: 0.3 # Weight for the math model
- model: Qwen/Qwen2.5-Coder-1.5B-Instruct # Code-focused model
parameters:
density: 0.6 # Retain 60% of significant parameters
weight: 0.2 # Weight for the coder model
parameters:
normalize: true # Normalize weights to ensure compatibility
int8_mask: true # Optimize memory and computational efficiency |