Hugging Face's logo

amy-hyunji-lee
/
olmo_base.step2384-unsharded

Model card Files Community
amy-hyunji-lee commited on
Commit
1eacf48
·
verified ·
1 Parent(s): edfa910

Upload folder using huggingface_hub

Browse files
Files changed (4) hide show
  1. config.yaml +447 -0
  2. model.pt +3 -0
  3. optim.pt +3 -0
  4. train.pt +3 -0
config.yaml ADDED
@@ -0,0 +1,447 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ run_name: olmo-400M-base
2
+ seed: 6198
3
+ epoch: null
4
+ dry_run: false
5
+ model:
6
+ d_model: 1024
7
+ n_heads: 16
8
+ n_kv_heads: null
9
+ clip_qkv: null
10
+ n_layers: 20
11
+ mlp_ratio: 8
12
+ mlp_hidden_size: null
13
+ activation_type: swiglu
14
+ block_type: sequential
15
+ block_group_size: 1
16
+ alibi: false
17
+ alibi_bias_max: 8.0
18
+ rope: true
19
+ rope_full_precision: true
20
+ rope_theta: 10000
21
+ flash_attention: true
22
+ attention_dropout: 0.0
23
+ multi_query_attention: null
24
+ attention_layer_norm: false
25
+ residual_dropout: 0.0
26
+ embedding_dropout: 0.0
27
+ embedding_layer_norm: false
28
+ layer_norm_type: rms
29
+ layer_norm_with_affine: true
30
+ layer_norm_eps: 1.0e-06
31
+ attention_layer_norm_with_affine: false
32
+ max_sequence_length: 2048
33
+ include_bias: false
34
+ bias_for_layer_norm: false
35
+ scale_logits: false
36
+ vocab_size: 50280
37
+ embedding_size: 50304
38
+ weight_tying: false
39
+ eos_token_id: 0
40
+ pad_token_id: 1
41
+ init_device: cuda
42
+ init_fn: normal
43
+ init_std: 0.02
44
+ init_cutoff_factor: 3.0
45
+ precision: amp_bf16
46
+ scale_emb_init: false
47
+ emb_init_std: null
48
+ norm_after: false
49
+ cbp_train: false
50
+ cbp_replacement_rate: 0.0001
51
+ cbp_maturity_threshold: 100
52
+ cbp_init_fn: kaiming
53
+ optimizer:
54
+ name: adamw
55
+ learning_rate: 0.0004
56
+ weight_decay: 0.1
57
+ betas:
58
+ - 0.9
59
+ - 0.95
60
+ eps: 1.0e-08
61
+ no_decay_norm_and_bias: null
62
+ selective_updates: false
63
+ decay_norm_and_bias: true
64
+ decay_embeddings: true
65
+ metrics_log_interval: 10
66
+ record_update_metrics: false
67
+ scheduler:
68
+ name: cosine_with_warmup
69
+ units: steps
70
+ t_warmup: 2384
71
+ t_max: null
72
+ alpha_f: 0.1
73
+ grad_clip_warmup_steps: null
74
+ grad_clip_warmup_factor: null
75
+ warmup_min_lr: 0.0
76
+ data:
77
+ paths:
78
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/000_00000-part-0-00000.npy
79
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/000_00001-part-0-00000.npy
80
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/000_00002-part-0-00000.npy
81
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/000_00003-part-0-00000.npy
82
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/000_00004-part-0-00000.npy
83
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/000_00005-part-0-00000.npy
84
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/000_00006-part-0-00000.npy
85
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/000_00007-part-0-00000.npy
86
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/000_00008-part-0-00000.npy
87
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/000_00009-part-0-00000.npy
88
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/001_00000-part-0-00000.npy
89
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/001_00001-part-0-00000.npy
90
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/001_00002-part-0-00000.npy
91
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/001_00003-part-0-00000.npy
92
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/001_00004-part-0-00000.npy
93
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/001_00005-part-0-00000.npy
94
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/001_00006-part-0-00000.npy
95
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/001_00007-part-0-00000.npy
96
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/001_00008-part-0-00000.npy
97
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/001_00009-part-0-00000.npy
98
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/002_00000-part-0-00000.npy
99
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/002_00001-part-0-00000.npy
100
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/002_00002-part-0-00000.npy
101
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/002_00003-part-0-00000.npy
102
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/002_00004-part-0-00000.npy
103
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/002_00005-part-0-00000.npy
104
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/002_00006-part-0-00000.npy
105
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/002_00007-part-0-00000.npy
106
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/002_00008-part-0-00000.npy
107
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/002_00009-part-0-00000.npy
108
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/003_00000-part-0-00000.npy
109
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/003_00001-part-0-00000.npy
110
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/003_00002-part-0-00000.npy
111
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/003_00003-part-0-00000.npy
112
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/003_00004-part-0-00000.npy
113
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/003_00005-part-0-00000.npy
114
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/003_00006-part-0-00000.npy
115
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/003_00007-part-0-00000.npy
116
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/003_00008-part-0-00000.npy
117
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/003_00009-part-0-00000.npy
118
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/004_00000-part-0-00000.npy
119
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/004_00001-part-0-00000.npy
120
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/004_00002-part-0-00000.npy
121
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/004_00003-part-0-00000.npy
122
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/004_00004-part-0-00000.npy
123
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/004_00005-part-0-00000.npy
124
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/004_00006-part-0-00000.npy
125
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/004_00007-part-0-00000.npy
126
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/004_00008-part-0-00000.npy
127
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/004_00009-part-0-00000.npy
128
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/005_00000-part-0-00000.npy
129
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/005_00001-part-0-00000.npy
130
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/005_00002-part-0-00000.npy
131
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/005_00003-part-0-00000.npy
132
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/005_00004-part-0-00000.npy
133
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/005_00005-part-0-00000.npy
134
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/005_00006-part-0-00000.npy
135
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/005_00007-part-0-00000.npy
136
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/005_00008-part-0-00000.npy
137
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/005_00009-part-0-00000.npy
138
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/006_00000-part-0-00000.npy
139
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/006_00001-part-0-00000.npy
140
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/006_00002-part-0-00000.npy
141
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/006_00003-part-0-00000.npy
142
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/006_00004-part-0-00000.npy
143
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/006_00005-part-0-00000.npy
144
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/006_00006-part-0-00000.npy
145
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/006_00007-part-0-00000.npy
146
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/006_00008-part-0-00000.npy
147
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/006_00009-part-0-00000.npy
148
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/007_00000-part-0-00000.npy
149
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/007_00001-part-0-00000.npy
150
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/007_00002-part-0-00000.npy
151
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/007_00003-part-0-00000.npy
152
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/007_00004-part-0-00000.npy
153
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/007_00005-part-0-00000.npy
154
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/007_00006-part-0-00000.npy
155
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/007_00007-part-0-00000.npy
156
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/007_00008-part-0-00000.npy
157
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/007_00009-part-0-00000.npy
158
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/008_00000-part-0-00000.npy
159
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/008_00001-part-0-00000.npy
160
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/008_00002-part-0-00000.npy
161
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/008_00003-part-0-00000.npy
162
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/008_00004-part-0-00000.npy
163
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/008_00005-part-0-00000.npy
164
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/008_00006-part-0-00000.npy
165
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/008_00007-part-0-00000.npy
166
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/008_00008-part-0-00000.npy
167
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/008_00009-part-0-00000.npy
168
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/009_00000-part-0-00000.npy
169
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/009_00001-part-0-00000.npy
170
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/009_00002-part-0-00000.npy
171
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/009_00003-part-0-00000.npy
172
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/009_00004-part-0-00000.npy
173
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/009_00005-part-0-00000.npy
174
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/009_00006-part-0-00000.npy
175
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/009_00007-part-0-00000.npy
176
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/009_00008-part-0-00000.npy
177
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/009_00009-part-0-00000.npy
178
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/010_00000-part-0-00000.npy
179
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/010_00001-part-0-00000.npy
180
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/010_00002-part-0-00000.npy
181
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/010_00003-part-0-00000.npy
182
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/010_00004-part-0-00000.npy
183
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/010_00005-part-0-00000.npy
184
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/010_00006-part-0-00000.npy
185
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/010_00007-part-0-00000.npy
186
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/010_00008-part-0-00000.npy
187
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/010_00009-part-0-00000.npy
188
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/011_00000-part-0-00000.npy
189
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/011_00001-part-0-00000.npy
190
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/011_00002-part-0-00000.npy
191
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/011_00003-part-0-00000.npy
192
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/011_00004-part-0-00000.npy
193
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/011_00005-part-0-00000.npy
194
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/011_00006-part-0-00000.npy
195
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/011_00007-part-0-00000.npy
196
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/011_00008-part-0-00000.npy
197
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/011_00009-part-0-00000.npy
198
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/012_00000-part-0-00000.npy
199
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/012_00001-part-0-00000.npy
200
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/012_00002-part-0-00000.npy
201
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/012_00003-part-0-00000.npy
202
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/012_00004-part-0-00000.npy
203
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/012_00005-part-0-00000.npy
204
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/012_00006-part-0-00000.npy
205
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/012_00007-part-0-00000.npy
206
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/012_00008-part-0-00000.npy
207
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/012_00009-part-0-00000.npy
208
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/013_00000-part-0-00000.npy
209
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/013_00001-part-0-00000.npy
210
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/013_00002-part-0-00000.npy
211
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/013_00003-part-0-00000.npy
212
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/013_00004-part-0-00000.npy
213
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/013_00005-part-0-00000.npy
214
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/013_00006-part-0-00000.npy
215
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/013_00007-part-0-00000.npy
216
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/013_00008-part-0-00000.npy
217
+ - /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/dolma/013_00009-part-0-00000.npy
218
+ memmap_dtype: uint16
219
+ datasets: null
220
+ dataset_path: null
221
+ label_mask_paths: null
222
+ pad_direction: right
223
+ generate_attention_mask: false
224
+ generate_doc_lengths: false
225
+ num_workers: 32
226
+ drop_last: true
227
+ pin_memory: true
228
+ prefetch_factor: 8
229
+ persistent_workers: true
230
+ timeout: 0
231
+ seed: null
232
+ instance_filter:
233
+ repetition_max_period: 13
234
+ repetition_min_period: 1
235
+ repetition_max_count: 32
236
+ restore_dataloader: true
237
+ fast_forward_batches: null
238
+ evaluators:
239
+ - label: piqa
240
+ type: downstream
241
+ data:
242
+ paths: null
243
+ memmap_dtype: uint16
244
+ datasets: null
245
+ dataset_path: null
246
+ label_mask_paths: null
247
+ pad_direction: right
248
+ generate_attention_mask: false
249
+ generate_doc_lengths: false
250
+ num_workers: 0
251
+ drop_last: false
252
+ pin_memory: false
253
+ prefetch_factor: null
254
+ persistent_workers: false
255
+ timeout: 0
256
+ seed: null
257
+ instance_filter: null
258
+ device_eval_batch_size: null
259
+ subset_num_batches: null
260
+ - label: hellaswag
261
+ type: downstream
262
+ data:
263
+ paths: null
264
+ memmap_dtype: uint16
265
+ datasets: null
266
+ dataset_path: null
267
+ label_mask_paths: null
268
+ pad_direction: right
269
+ generate_attention_mask: false
270
+ generate_doc_lengths: false
271
+ num_workers: 0
272
+ drop_last: false
273
+ pin_memory: false
274
+ prefetch_factor: null
275
+ persistent_workers: false
276
+ timeout: 0
277
+ seed: null
278
+ instance_filter: null
279
+ device_eval_batch_size: null
280
+ subset_num_batches: null
281
+ - label: sciq
282
+ type: downstream
283
+ data:
284
+ paths: null
285
+ memmap_dtype: uint16
286
+ datasets: null
287
+ dataset_path: null
288
+ label_mask_paths: null
289
+ pad_direction: right
290
+ generate_attention_mask: false
291
+ generate_doc_lengths: false
292
+ num_workers: 0
293
+ drop_last: false
294
+ pin_memory: false
295
+ prefetch_factor: null
296
+ persistent_workers: false
297
+ timeout: 0
298
+ seed: null
299
+ instance_filter: null
300
+ device_eval_batch_size: null
301
+ subset_num_batches: null
302
+ - label: arc_easy
303
+ type: downstream
304
+ data:
305
+ paths: null
306
+ memmap_dtype: uint16
307
+ datasets: null
308
+ dataset_path: null
309
+ label_mask_paths: null
310
+ pad_direction: right
311
+ generate_attention_mask: false
312
+ generate_doc_lengths: false
313
+ num_workers: 0
314
+ drop_last: false
315
+ pin_memory: false
316
+ prefetch_factor: null
317
+ persistent_workers: false
318
+ timeout: 0
319
+ seed: null
320
+ instance_filter: null
321
+ device_eval_batch_size: null
322
+ subset_num_batches: null
323
+ - label: mrpc
324
+ type: downstream
325
+ data:
326
+ paths: null
327
+ memmap_dtype: uint16
328
+ datasets: null
329
+ dataset_path: null
330
+ label_mask_paths: null
331
+ pad_direction: right
332
+ generate_attention_mask: false
333
+ generate_doc_lengths: false
334
+ num_workers: 0
335
+ drop_last: false
336
+ pin_memory: false
337
+ prefetch_factor: null
338
+ persistent_workers: false
339
+ timeout: 0
340
+ seed: null
341
+ instance_filter: null
342
+ device_eval_batch_size: null
343
+ subset_num_batches: null
344
+ - label: sst2
345
+ type: downstream
346
+ data:
347
+ paths: null
348
+ memmap_dtype: uint16
349
+ datasets: null
350
+ dataset_path: null
351
+ label_mask_paths: null
352
+ pad_direction: right
353
+ generate_attention_mask: false
354
+ generate_doc_lengths: false
355
+ num_workers: 0
356
+ drop_last: false
357
+ pin_memory: false
358
+ prefetch_factor: null
359
+ persistent_workers: false
360
+ timeout: 0
361
+ seed: null
362
+ instance_filter: null
363
+ device_eval_batch_size: null
364
+ subset_num_batches: null
365
+ eval_interval: 2384
366
+ tokenizer:
367
+ identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json
368
+ truncate_direction: right
369
+ save_folder: /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/workspace/olmo-400M-base
370
+ remote_save_folder: null
371
+ canceled_check_interval: 50
372
+ save_interval: 2384
373
+ save_interval_unsharded: 2384
374
+ save_interval_ephemeral: null
375
+ save_num_checkpoints_to_keep: 2
376
+ save_num_unsharded_checkpoints_to_keep: -1
377
+ save_overwrite: true
378
+ force_save_unsharded: false
379
+ no_pre_train_checkpoint: false
380
+ load_path: /apdcephfs_sh2/share_300000800/user/kaixinma/amylee/fineweb-edu/workspace/olmo-400M-base/latest-unsharded
381
+ load_path_sharded_checkpointer: null
382
+ try_load_latest_save: false
383
+ reset_optimizer_state: false
384
+ reset_trainer_state: false
385
+ sharded_checkpointer: torch_legacy
386
+ new_style_checkpoints: null
387
+ max_duration: 1ep
388
+ global_train_batch_size: 1024
389
+ device_train_batch_size: 128
390
+ device_train_microbatch_size: 16
391
+ device_eval_batch_size: 16
392
+ eval_subset_num_batches: -1
393
+ eval_on_load: true
394
+ device_train_grad_accum: 8
395
+ max_grad_norm: 1.0
396
+ max_grad_norm_ratio: null
397
+ precision: amp_bf16
398
+ wandb:
399
+ project: olmo-pretrain-ablation
400
+ entity: alee6868
401
+ group: null
402
+ name: olmo-400M-base
403
+ tags:
404
+ - watching
405
+ log_artifacts: false
406
+ rank_zero_only: true
407
+ log_interval: 1
408
+ speed_monitor:
409
+ window_size: 20
410
+ gpu_flops_available: null
411
+ console_log_interval: 1
412
+ gen1_gc_interval: 1
413
+ compile: null
414
+ distributed_strategy: ddp
415
+ fsdp:
416
+ use_orig_params: true
417
+ sharding_strategy: FULL_SHARD
418
+ wrapping_strategy: null
419
+ precision: pure
420
+ hybrid_sharding_num_model_replicas: null
421
+ ddp:
422
+ grad_sync_mode: batch
423
+ find_unused_params: false
424
+ softmax_auxiliary_loss: false
425
+ auxiliary_loss_multiplier: 0.0001
426
+ time_limit: null
427
+ extra_steps_after_cancel: 10
428
+ early_stopping_factor: null
429
+ save_data_indices: true
430
+ python_profiling: false
431
+ torch_profiling: false
432
+ stop_at: 47680
433
+ stop_after: null
434
+ activation_checkpointing: null
435
+ fused_loss: null
436
+ inject_indices_map: null
437
+ inject_interval: null
438
+ resus_portion: 1.0
439
+ resus_ratio: 1.0
440
+ data_shuffling: true
441
+ KE_loss: false
442
+ sum_CE_KE_loss: true
443
+ lambda_ke_loss: null
444
+ grad_ascent: false
445
+ trainable_parameter: ''
446
+ hf_datasets_cache_dir: null
447
+ module_outputs_save_steps: null
model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f04719818b23eb010024dc0781cf73458238a3ccee2fbe119a7ec0e4dc6a8cd1
3
+ size 1754478590
optim.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:48bcfcb09b1b6df74c3334df0f7c87f3b9411d19ad3ce1ee0d9f1b99d165bcd4
3
+ size 3508964070
train.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:23f5e9cedf11b1d7277354ae04d5a321d817d4f1479c638c67943fb994731ebf
3
+ size 14668