{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "6e3c72a5", "metadata": { "execution": { "iopub.execute_input": "2024-10-18T11:48:16.063328Z", "iopub.status.busy": "2024-10-18T11:48:16.062890Z", "iopub.status.idle": "2024-10-18T11:48:17.059345Z", "shell.execute_reply": "2024-10-18T11:48:17.058073Z" }, "papermill": { "duration": 1.004239, "end_time": "2024-10-18T11:48:17.061662", "exception": false, "start_time": "2024-10-18T11:48:16.057423", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "!rm -rf /kaggle/working/*" ] }, { "cell_type": "code", "execution_count": 2, "id": "670bfb10", "metadata": { "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5", "execution": { "iopub.execute_input": "2024-10-18T11:48:17.070672Z", "iopub.status.busy": "2024-10-18T11:48:17.070323Z", "iopub.status.idle": "2024-10-18T11:48:21.684129Z", "shell.execute_reply": "2024-10-18T11:48:21.682796Z" }, "papermill": { "duration": 4.621045, "end_time": "2024-10-18T11:48:21.686673", "exception": false, "start_time": "2024-10-18T11:48:17.065628", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cloning into 'VALL_E_X'...\r\n", "remote: Enumerating objects: 230, done.\u001b[K\r\n", "remote: Counting objects: 100% (230/230), done.\u001b[K\r\n", "remote: Compressing objects: 100% (189/189), done.\u001b[K\r\n", "remote: Total 230 (delta 47), reused 214 (delta 31), pack-reused 0 (from 0)\u001b[K\r\n", "Receiving objects: 100% (230/230), 15.25 MiB | 26.12 MiB/s, done.\r\n", "Resolving deltas: 100% (47/47), done.\r\n", "renamed '/kaggle/working/VALL_E_X/LICENSE' -> '/kaggle/working/LICENSE'\r\n", "renamed '/kaggle/working/VALL_E_X/README.md' -> '/kaggle/working/README.md'\r\n", "renamed '/kaggle/working/VALL_E_X/customs' -> '/kaggle/working/customs'\r\n", "renamed '/kaggle/working/VALL_E_X/data' -> '/kaggle/working/data'\r\n", "renamed '/kaggle/working/VALL_E_X/descriptions.py' -> '/kaggle/working/descriptions.py'\r\n", "renamed '/kaggle/working/VALL_E_X/examples.py' -> '/kaggle/working/examples.py'\r\n", "renamed '/kaggle/working/VALL_E_X/exp' -> '/kaggle/working/exp'\r\n", "renamed '/kaggle/working/VALL_E_X/images' -> '/kaggle/working/images'\r\n", "renamed '/kaggle/working/VALL_E_X/infer.ipynb' -> '/kaggle/working/infer.ipynb'\r\n", "renamed '/kaggle/working/VALL_E_X/launch-ui.py' -> '/kaggle/working/launch-ui.py'\r\n", "renamed '/kaggle/working/VALL_E_X/macros.py' -> '/kaggle/working/macros.py'\r\n", "renamed '/kaggle/working/VALL_E_X/makedata.ipynb' -> '/kaggle/working/makedata.ipynb'\r\n", "renamed '/kaggle/working/VALL_E_X/model-card.md' -> '/kaggle/working/model-card.md'\r\n", "renamed '/kaggle/working/VALL_E_X/models' -> '/kaggle/working/models'\r\n", "renamed '/kaggle/working/VALL_E_X/modules' -> '/kaggle/working/modules'\r\n", "renamed '/kaggle/working/VALL_E_X/nltk_data' -> '/kaggle/working/nltk_data'\r\n", "renamed '/kaggle/working/VALL_E_X/presets' -> '/kaggle/working/presets'\r\n", "renamed '/kaggle/working/VALL_E_X/prompts' -> '/kaggle/working/prompts'\r\n", "renamed '/kaggle/working/VALL_E_X/requirements.txt' -> '/kaggle/working/requirements.txt'\r\n", "renamed '/kaggle/working/VALL_E_X/test.py' -> '/kaggle/working/test.py'\r\n", "renamed '/kaggle/working/VALL_E_X/train.py' -> '/kaggle/working/train.py'\r\n", "renamed '/kaggle/working/VALL_E_X/train_utils' -> '/kaggle/working/train_utils'\r\n", "renamed '/kaggle/working/VALL_E_X/utils' -> '/kaggle/working/utils'\r\n" ] } ], "source": [ "!git clone https://github.com/windymv025/VALL_E_X.git\n", "!mv -v /kaggle/working/VALL_E_X/* /kaggle/working/\n", "!rm -rf VALL_E_X" ] }, { "cell_type": "code", "execution_count": 3, "id": "aee02f01", "metadata": { "execution": { "iopub.execute_input": "2024-10-18T11:48:21.699113Z", "iopub.status.busy": "2024-10-18T11:48:21.698730Z", "iopub.status.idle": "2024-10-18T11:49:19.670857Z", "shell.execute_reply": "2024-10-18T11:49:19.669797Z" }, "papermill": { "duration": 57.981266, "end_time": "2024-10-18T11:49:19.673452", "exception": false, "start_time": "2024-10-18T11:48:21.692186", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "!pip install -q -r requirements.txt" ] }, { "cell_type": "code", "execution_count": 4, "id": "d6e3a184", "metadata": { "execution": { "iopub.execute_input": "2024-10-18T11:49:19.685851Z", "iopub.status.busy": "2024-10-18T11:49:19.685509Z", "iopub.status.idle": "2024-10-18T11:49:19.690359Z", "shell.execute_reply": "2024-10-18T11:49:19.689598Z" }, "papermill": { "duration": 0.013814, "end_time": "2024-10-18T11:49:19.692777", "exception": false, "start_time": "2024-10-18T11:49:19.678963", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "input_dataset_path = '/kaggle/input/vivos-vietnamese-speech-dataset-ljspeech-format/vivos'\n", "input_train_path = f'{input_dataset_path}/train'\n", "input_test_path = f'{input_dataset_path}/test'\n", "prompt_file_name = 'audio_ann_sum.txt'\n", "\n", "# dataset_path = 'vivos_datasets'\n", "# train_path = f'{dataset_path}/train'\n", "# test_path = f'{dataset_path}/test'" ] }, { "cell_type": "code", "execution_count": 5, "id": "9793eb64", "metadata": { "execution": { "iopub.execute_input": "2024-10-18T11:49:19.704857Z", "iopub.status.busy": "2024-10-18T11:49:19.704573Z", "iopub.status.idle": "2024-10-18T11:49:21.671192Z", "shell.execute_reply": "2024-10-18T11:49:21.670080Z" }, "papermill": { "duration": 1.975192, "end_time": "2024-10-18T11:49:21.673667", "exception": false, "start_time": "2024-10-18T11:49:19.698475", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mkdir: missing operand\r\n", "Try 'mkdir --help' for more information.\r\n", "mkdir: missing operand\r\n", "Try 'mkdir --help' for more information.\r\n" ] } ], "source": [ "!mkdir -p $train_path\n", "!mkdir -p $test_path" ] }, { "cell_type": "code", "execution_count": 6, "id": "ff3ece1b", "metadata": { "execution": { "iopub.execute_input": "2024-10-18T11:49:21.687532Z", "iopub.status.busy": "2024-10-18T11:49:21.687180Z", "iopub.status.idle": "2024-10-18T11:49:26.076048Z", "shell.execute_reply": "2024-10-18T11:49:26.075068Z" }, "papermill": { "duration": 4.398011, "end_time": "2024-10-18T11:49:26.078461", "exception": false, "start_time": "2024-10-18T11:49:21.680450", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "import shutil\n", "import os\n", "import h5py\n", "import glob\n", "import torch\n", "import torchaudio\n", "\n", "# import torch_xla\n", "# import torch_xla.core.xla_model as xm\n", "\n", "# print(f'PyTorch can access {xm.xla_device()} TPU cores')\n", "# tpu_device = xm.xla_device()\n", "\n", "# torch_xla.device_count()" ] }, { "cell_type": "code", "execution_count": 7, "id": "900540c2", "metadata": { "execution": { "iopub.execute_input": "2024-10-18T11:49:26.091112Z", "iopub.status.busy": "2024-10-18T11:49:26.090666Z", "iopub.status.idle": "2024-10-18T11:50:00.674322Z", "shell.execute_reply": "2024-10-18T11:50:00.673279Z" }, "papermill": { "duration": 34.592637, "end_time": "2024-10-18T11:50:00.676891", "exception": false, "start_time": "2024-10-18T11:49:26.084254", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Downloading...\r\n", "From (original): https://drive.google.com/uc?id=10gdQWvP-K_e1undkvv0p2b7SU6I4Egyl\r\n", "From (redirected): https://drive.google.com/uc?id=10gdQWvP-K_e1undkvv0p2b7SU6I4Egyl&confirm=t&uuid=da3af224-2df8-4b61-b5f0-74a8f0598aef\r\n", "To: /kaggle/working/vallex-checkpoint.pt\r\n", "100%|██████████████████████████████████████| 1.48G/1.48G [00:16<00:00, 89.8MB/s]\r\n" ] } ], "source": [ "!pip install -q gdown\n", "!gdown 10gdQWvP-K_e1undkvv0p2b7SU6I4Egyl" ] }, { "cell_type": "code", "execution_count": 8, "id": "06666c0f", "metadata": { "execution": { "iopub.execute_input": "2024-10-18T11:50:00.704880Z", "iopub.status.busy": "2024-10-18T11:50:00.704061Z", "iopub.status.idle": "2024-10-18T11:50:04.862134Z", "shell.execute_reply": "2024-10-18T11:50:04.860704Z" }, "papermill": { "duration": 4.174612, "end_time": "2024-10-18T11:50:04.864561", "exception": false, "start_time": "2024-10-18T11:50:00.689949", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "!mkdir -p checkpoints\n", "!cp vallex-checkpoint.pt checkpoints/vallex-checkpoint_modified.pt\n", "!mv vallex-checkpoint.pt checkpoints/vallex-checkpoint.pt" ] }, { "cell_type": "code", "execution_count": 9, "id": "a110952f", "metadata": { "execution": { "iopub.execute_input": "2024-10-18T11:50:04.893717Z", "iopub.status.busy": "2024-10-18T11:50:04.893349Z", "iopub.status.idle": "2024-10-18T11:50:04.897543Z", "shell.execute_reply": "2024-10-18T11:50:04.896741Z" }, "papermill": { "duration": 0.02051, "end_time": "2024-10-18T11:50:04.899480", "exception": false, "start_time": "2024-10-18T11:50:04.878970", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "# !python -X utf8 launch-ui.py\n", "# --keep-last-k 2 \\" ] }, { "cell_type": "code", "execution_count": 10, "id": "e5244bba", "metadata": { "execution": { "iopub.execute_input": "2024-10-18T11:50:04.925704Z", "iopub.status.busy": "2024-10-18T11:50:04.925397Z", "iopub.status.idle": "2024-10-18T13:49:31.548142Z", "shell.execute_reply": "2024-10-18T13:49:31.546830Z" }, "papermill": { "duration": 7166.640325, "end_time": "2024-10-18T13:49:31.552320", "exception": false, "start_time": "2024-10-18T11:50:04.911995", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Operating System: Linux\r\n", "Downloading: \"https://dl.fbaipublicfiles.com/encodec/v0/encodec_24khz-d7cc33bc.th\" to /root/.cache/torch/hub/checkpoints/encodec_24khz-d7cc33bc.th\r\n", "100%|███████████████████████████████████████| 88.9M/88.9M [00:00<00:00, 179MB/s]\r\n", "2024-10-18 11:50:35,401 INFO [train.py:861] Training started\r\n", "2024-10-18 11:50:35,402 INFO [train.py:880] Device: cuda:0\r\n", "2024-10-18 11:50:35,403 INFO [train.py:881] {'best_train_loss': inf, 'best_valid_loss': inf, 'best_train_epoch': -1, 'best_valid_epoch': -1, 'batch_idx_train': 0, 'log_interval': 100, 'reset_interval': 200, 'valid_interval': 10000, 'world_size': 1, 'master_port': 12354, 'tensorboard': True, 'num_epochs': 100, 'start_epoch': 1, 'start_batch': 0, 'exp_dir': PosixPath('exp/valle_dev'), 'optimizer_name': 'ScaledAdam', 'scheduler_name': 'Eden', 'base_lr': 0.005, 'warmup_steps': 200, 'seed': 42, 'inf_check': False, 'save_every_n': 10, 'keep_last_k': 2, 'average_period': 0, 'accumulate_grad_steps': 1, 'dtype': 'bfloat16', 'filter_min_duration': 0.0, 'filter_max_duration': 20.0, 'train_stage': 0, 'visualize': True, 'oom_check': True, 'train_dir': '/kaggle/input/vivos-vietnamese-speech-dataset-ljspeech-format/vivos/train', 'valid_dir': '/kaggle/input/vivos-vietnamese-speech-dataset-ljspeech-format/vivos/test', 'checkpoint_path': None, 'model_name': 'VALL-E', 'decoder_dim': 1024, 'nhead': 16, 'num_decoder_layers': 12, 'scale_factor': 1.0, 'norm_first': True, 'add_prenet': False, 'prefix_mode': 0, 'share_embedding': True, 'prepend_bos': False, 'num_quantizers': 8, 'scaling_xformers': False}\r\n", "2024-10-18 11:50:35,403 INFO [train.py:883] About to create model\r\n", "config.yaml: 100%|█████████████████████████████| 503/503 [00:00<00:00, 3.24MB/s]\r\n", "pytorch_model.bin: 100%|████████████████████| 40.4M/40.4M [00:00<00:00, 174MB/s]\r\n", "2024-10-18 11:50:39,524 INFO [train.py:887] Number of model parameters: 370539524\r\n", "2024-10-18 11:50:51,676 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-10.pt\r\n", "2024-10-18 11:51:09,403 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-20.pt\r\n", "2024-10-18 11:51:27,320 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-30.pt\r\n", "2024-10-18 11:51:45,896 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-40.pt\r\n", "2024-10-18 11:52:08,375 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-50.pt\r\n", "2024-10-18 11:52:33,732 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-60.pt\r\n", "2024-10-18 11:52:52,032 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-70.pt\r\n", "2024-10-18 11:53:10,370 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-80.pt\r\n", "2024-10-18 11:53:28,424 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-90.pt\r\n", "2024-10-18 11:53:56,977 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-100.pt\r\n", "2024-10-18 11:54:06,210 INFO [train.py:774] Epoch 1, batch 100, train_loss[loss=3.156, ArTop10Accuracy=0.7427, NarTop10Accuracy=0.6508, over 991.00 frames. ], tot_loss[loss=3.339, ArTop10Accuracy=0.7312, NarTop10Accuracy=0.5875, over 444.10 frames. ], batch size: 4, lr: 3.75e-03\r\n", "2024-10-18 11:54:15,313 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-110.pt\r\n", "2024-10-18 11:54:43,755 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-120.pt\r\n", "2024-10-18 11:55:02,218 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-130.pt\r\n", "2024-10-18 11:55:20,752 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-140.pt\r\n", "2024-10-18 11:55:39,342 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-150.pt\r\n", "2024-10-18 11:55:58,052 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-160.pt\r\n", "2024-10-18 11:56:18,573 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-170.pt\r\n", "2024-10-18 11:56:36,762 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-180.pt\r\n", "2024-10-18 11:56:55,045 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-190.pt\r\n", "2024-10-18 11:57:17,998 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-200.pt\r\n", "2024-10-18 11:57:26,861 INFO [train.py:774] Epoch 1, batch 200, train_loss[loss=3.178, ArTop10Accuracy=0.7536, NarTop10Accuracy=0.5949, over 1319.00 frames. ], tot_loss[loss=3.276, ArTop10Accuracy=0.7475, NarTop10Accuracy=0.5947, over 720.18 frames. ], batch size: 5, lr: 5.00e-03\r\n", "2024-10-18 11:57:35,984 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-210.pt\r\n", "2024-10-18 11:57:57,868 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-220.pt\r\n", "2024-10-18 11:58:25,855 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-230.pt\r\n", "2024-10-18 11:58:44,087 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-240.pt\r\n", "2024-10-18 11:59:02,328 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-250.pt\r\n", "2024-10-18 11:59:24,312 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-260.pt\r\n", "2024-10-18 11:59:42,035 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-270.pt\r\n", "2024-10-18 12:00:02,784 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-280.pt\r\n", "2024-10-18 12:00:21,521 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-290.pt\r\n", "2024-10-18 12:00:39,680 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-300.pt\r\n", "2024-10-18 12:00:51,542 INFO [train.py:774] Epoch 1, batch 300, train_loss[loss=3.129, ArTop10Accuracy=0.8019, NarTop10Accuracy=0.6124, over 1141.00 frames. ], tot_loss[loss=3.284, ArTop10Accuracy=0.7595, NarTop10Accuracy=0.5824, over 899.16 frames. ], batch size: 3, lr: 5.00e-03\r\n", "2024-10-18 12:01:10,918 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-310.pt\r\n", "2024-10-18 12:01:29,341 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-320.pt\r\n", "2024-10-18 12:01:47,348 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-330.pt\r\n", "2024-10-18 12:02:08,142 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-340.pt\r\n", "2024-10-18 12:02:30,763 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-350.pt\r\n", "2024-10-18 12:02:51,610 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-360.pt\r\n", "2024-10-18 12:03:12,278 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-370.pt\r\n", "2024-10-18 12:03:31,046 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-380.pt\r\n", "2024-10-18 12:03:49,715 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-390.pt\r\n", "2024-10-18 12:04:10,058 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-400.pt\r\n", "2024-10-18 12:04:23,527 INFO [train.py:774] Epoch 1, batch 400, train_loss[loss=3.847, ArTop10Accuracy=0.7003, NarTop10Accuracy=0.4367, over 1161.00 frames. ], tot_loss[loss=3.301, ArTop10Accuracy=0.7644, NarTop10Accuracy=0.5705, over 1001.79 frames. ], batch size: 2, lr: 4.99e-03\r\n", "2024-10-18 12:04:33,233 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-410.pt\r\n", "2024-10-18 12:04:50,999 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-420.pt\r\n", "2024-10-18 12:05:09,454 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-430.pt\r\n", "2024-10-18 12:05:31,430 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-440.pt\r\n", "2024-10-18 12:05:52,910 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-450.pt\r\n", "2024-10-18 12:06:15,441 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-460.pt\r\n", "2024-10-18 12:06:33,508 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-470.pt\r\n", "2024-10-18 12:06:55,605 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-480.pt\r\n", "2024-10-18 12:07:17,199 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-490.pt\r\n", "2024-10-18 12:07:42,724 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-500.pt\r\n", "2024-10-18 12:07:52,702 INFO [train.py:774] Epoch 1, batch 500, train_loss[loss=2.661, ArTop10Accuracy=0.8497, NarTop10Accuracy=0.6367, over 858.00 frames. ], tot_loss[loss=3.297, ArTop10Accuracy=0.7702, NarTop10Accuracy=0.5653, over 1055.81 frames. ], batch size: 1, lr: 4.99e-03\r\n", "2024-10-18 12:08:01,968 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-510.pt\r\n", "2024-10-18 12:08:32,336 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-520.pt\r\n", "2024-10-18 12:08:51,493 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-530.pt\r\n", "2024-10-18 12:09:10,769 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-540.pt\r\n", "2024-10-18 12:09:29,322 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-550.pt\r\n", "2024-10-18 12:09:47,394 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-560.pt\r\n", "2024-10-18 12:10:05,328 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-570.pt\r\n", "2024-10-18 12:10:23,755 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-580.pt\r\n", "2024-10-18 12:10:45,250 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-590.pt\r\n", "2024-10-18 12:11:05,189 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-600.pt\r\n", "2024-10-18 12:11:17,042 INFO [train.py:774] Epoch 1, batch 600, train_loss[loss=3.634, ArTop10Accuracy=0.7451, NarTop10Accuracy=0.4745, over 1334.00 frames. ], tot_loss[loss=3.297, ArTop10Accuracy=0.773, NarTop10Accuracy=0.5634, over 1088.47 frames. ], batch size: 5, lr: 4.98e-03\r\n", "2024-10-18 12:11:25,469 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-610.pt\r\n", "2024-10-18 12:11:44,686 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-620.pt\r\n", "2024-10-18 12:12:13,647 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-630.pt\r\n", "2024-10-18 12:12:32,573 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-640.pt\r\n", "2024-10-18 12:12:51,311 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-650.pt\r\n", "2024-10-18 12:13:09,299 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-660.pt\r\n", "2024-10-18 12:13:27,208 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-670.pt\r\n", "2024-10-18 12:13:48,102 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-680.pt\r\n", "2024-10-18 12:14:06,664 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-690.pt\r\n", "2024-10-18 12:14:38,245 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-700.pt\r\n", "2024-10-18 12:14:47,729 INFO [train.py:774] Epoch 1, batch 700, train_loss[loss=3.533, ArTop10Accuracy=0.7197, NarTop10Accuracy=0.534, over 1295.00 frames. ], tot_loss[loss=3.283, ArTop10Accuracy=0.7767, NarTop10Accuracy=0.5662, over 1116.67 frames. ], batch size: 3, lr: 4.98e-03\r\n", "2024-10-18 12:14:56,907 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-710.pt\r\n", "2024-10-18 12:15:14,678 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-720.pt\r\n", "2024-10-18 12:15:34,418 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-730.pt\r\n", "2024-10-18 12:16:04,540 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-740.pt\r\n", "2024-10-18 12:16:24,713 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-750.pt\r\n", "2024-10-18 12:16:56,082 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-760.pt\r\n", "2024-10-18 12:17:14,990 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-770.pt\r\n", "2024-10-18 12:17:34,199 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-780.pt\r\n", "2024-10-18 12:17:53,479 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-790.pt\r\n", "2024-10-18 12:18:11,992 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-800.pt\r\n", "2024-10-18 12:18:21,072 INFO [train.py:774] Epoch 1, batch 800, train_loss[loss=3.517, ArTop10Accuracy=0.7732, NarTop10Accuracy=0.4902, over 1054.00 frames. ], tot_loss[loss=3.275, ArTop10Accuracy=0.7812, NarTop10Accuracy=0.565, over 1138.05 frames. ], batch size: 4, lr: 4.97e-03\r\n", "2024-10-18 12:18:30,704 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-810.pt\r\n", "2024-10-18 12:18:48,903 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-820.pt\r\n", "2024-10-18 12:19:06,523 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-830.pt\r\n", "2024-10-18 12:19:26,557 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-840.pt\r\n", "2024-10-18 12:19:45,502 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-850.pt\r\n", "2024-10-18 12:20:04,069 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-860.pt\r\n", "2024-10-18 12:20:22,036 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-870.pt\r\n", "2024-10-18 12:20:54,976 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-880.pt\r\n", "2024-10-18 12:21:13,593 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-890.pt\r\n", "2024-10-18 12:21:32,563 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-900.pt\r\n", "2024-10-18 12:21:41,335 INFO [train.py:774] Epoch 1, batch 900, train_loss[loss=3.443, ArTop10Accuracy=0.7763, NarTop10Accuracy=0.5095, over 1323.00 frames. ], tot_loss[loss=3.282, ArTop10Accuracy=0.7819, NarTop10Accuracy=0.5621, over 1136.99 frames. ], batch size: 6, lr: 4.96e-03\r\n", "2024-10-18 12:21:51,764 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-910.pt\r\n", "2024-10-18 12:22:22,454 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-920.pt\r\n", "2024-10-18 12:22:40,903 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-930.pt\r\n", "2024-10-18 12:22:59,821 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-940.pt\r\n", "2024-10-18 12:23:19,585 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-950.pt\r\n", "2024-10-18 12:23:38,571 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-960.pt\r\n", "2024-10-18 12:23:59,206 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-970.pt\r\n", "2024-10-18 12:24:19,152 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-980.pt\r\n", "2024-10-18 12:24:37,915 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-990.pt\r\n", "2024-10-18 12:24:57,044 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1000.pt\r\n", "2024-10-18 12:25:06,750 INFO [train.py:774] Epoch 1, batch 1000, train_loss[loss=3.65, ArTop10Accuracy=0.7215, NarTop10Accuracy=0.5005, over 1185.00 frames. ], tot_loss[loss=3.298, ArTop10Accuracy=0.7809, NarTop10Accuracy=0.5582, over 1152.85 frames. ], batch size: 5, lr: 4.95e-03\r\n", "2024-10-18 12:25:08,028 INFO [utils.py:877] Clipping_scale=2.0, grad-norm quartiles 3.245e+01 5.003e+01 5.612e+01 6.476e+01 1.276e+02, threshold=1.122e+02, percent-clipped=0.0\r\n", "2024-10-18 12:25:28,894 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1010.pt\r\n", "2024-10-18 12:25:48,507 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1020.pt\r\n", "2024-10-18 12:26:07,339 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1030.pt\r\n", "2024-10-18 12:26:25,959 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1040.pt\r\n", "2024-10-18 12:26:55,262 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1050.pt\r\n", "2024-10-18 12:27:14,875 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1060.pt\r\n", "2024-10-18 12:27:33,949 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1070.pt\r\n", "2024-10-18 12:27:57,193 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1080.pt\r\n", "2024-10-18 12:28:19,973 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1090.pt\r\n", "2024-10-18 12:28:37,820 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1100.pt\r\n", "2024-10-18 12:28:50,034 INFO [train.py:774] Epoch 1, batch 1100, train_loss[loss=3.756, ArTop10Accuracy=0.7481, NarTop10Accuracy=0.4023, over 1199.00 frames. ], tot_loss[loss=3.299, ArTop10Accuracy=0.7824, NarTop10Accuracy=0.5568, over 1162.96 frames. ], batch size: 3, lr: 4.94e-03\r\n", "2024-10-18 12:29:00,331 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1110.pt\r\n", "2024-10-18 12:29:24,861 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1120.pt\r\n", "2024-10-18 12:29:43,679 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1130.pt\r\n", "2024-10-18 12:30:12,036 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1140.pt\r\n", "2024-10-18 12:30:30,831 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1150.pt\r\n", "2024-10-18 12:30:49,142 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1160.pt\r\n", "2024-10-18 12:31:11,528 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1170.pt\r\n", "2024-10-18 12:31:33,167 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1180.pt\r\n", "2024-10-18 12:31:55,606 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1190.pt\r\n", "2024-10-18 12:32:18,897 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1200.pt\r\n", "2024-10-18 12:32:28,516 INFO [train.py:774] Epoch 1, batch 1200, train_loss[loss=3.386, ArTop10Accuracy=0.7247, NarTop10Accuracy=0.6049, over 1246.00 frames. ], tot_loss[loss=3.286, ArTop10Accuracy=0.7832, NarTop10Accuracy=0.5599, over 1157.58 frames. ], batch size: 3, lr: 4.93e-03\r\n", "2024-10-18 12:32:37,427 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1210.pt\r\n", "2024-10-18 12:32:56,564 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1220.pt\r\n", "2024-10-18 12:33:16,694 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1230.pt\r\n", "2024-10-18 12:33:46,781 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1240.pt\r\n", "2024-10-18 12:34:05,399 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1250.pt\r\n", "2024-10-18 12:34:32,676 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1260.pt\r\n", "2024-10-18 12:34:51,303 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1270.pt\r\n", "2024-10-18 12:35:09,644 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1280.pt\r\n", "2024-10-18 12:35:28,364 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1290.pt\r\n", "2024-10-18 12:35:49,446 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1300.pt\r\n", "2024-10-18 12:35:58,720 INFO [train.py:774] Epoch 1, batch 1300, train_loss[loss=2.983, ArTop10Accuracy=0.8195, NarTop10Accuracy=0.6625, over 1202.00 frames. ], tot_loss[loss=3.287, ArTop10Accuracy=0.7828, NarTop10Accuracy=0.5606, over 1161.95 frames. ], batch size: 5, lr: 4.92e-03\r\n", "2024-10-18 12:36:08,326 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1310.pt\r\n", "2024-10-18 12:36:34,656 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1320.pt\r\n", "2024-10-18 12:36:53,403 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1330.pt\r\n", "2024-10-18 12:37:12,072 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1340.pt\r\n", "2024-10-18 12:37:30,308 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1350.pt\r\n", "2024-10-18 12:38:06,730 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1360.pt\r\n", "2024-10-18 12:38:25,110 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1370.pt\r\n", "2024-10-18 12:38:44,039 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1380.pt\r\n", "2024-10-18 12:39:02,893 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1390.pt\r\n", "2024-10-18 12:39:20,715 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1400.pt\r\n", "2024-10-18 12:39:29,924 INFO [train.py:774] Epoch 1, batch 1400, train_loss[loss=3.524, ArTop10Accuracy=0.741, NarTop10Accuracy=0.5082, over 1112.00 frames. ], tot_loss[loss=3.287, ArTop10Accuracy=0.7829, NarTop10Accuracy=0.5594, over 1152.10 frames. ], batch size: 4, lr: 4.91e-03\r\n", "2024-10-18 12:39:38,786 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1410.pt\r\n", "2024-10-18 12:39:57,127 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1420.pt\r\n", "2024-10-18 12:40:18,678 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1430.pt\r\n", "2024-10-18 12:40:41,099 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1440.pt\r\n", "2024-10-18 12:40:59,509 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1450.pt\r\n", "2024-10-18 12:41:24,433 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1460.pt\r\n", "2024-10-18 12:41:43,381 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1470.pt\r\n", "2024-10-18 12:42:02,083 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1480.pt\r\n", "2024-10-18 12:42:21,170 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1490.pt\r\n", "2024-10-18 12:42:39,813 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1500.pt\r\n", "2024-10-18 12:42:49,093 INFO [train.py:774] Epoch 1, batch 1500, train_loss[loss=2.999, ArTop10Accuracy=0.8037, NarTop10Accuracy=0.6299, over 1228.00 frames. ], tot_loss[loss=3.272, ArTop10Accuracy=0.7865, NarTop10Accuracy=0.5621, over 1148.42 frames. ], batch size: 5, lr: 4.89e-03\r\n", "2024-10-18 12:42:58,447 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1510.pt\r\n", "2024-10-18 12:43:17,270 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1520.pt\r\n", "2024-10-18 12:43:35,191 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1530.pt\r\n", "2024-10-18 12:43:54,065 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1540.pt\r\n", "2024-10-18 12:44:12,620 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1550.pt\r\n", "2024-10-18 12:44:31,212 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1560.pt\r\n", "2024-10-18 12:44:49,152 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1570.pt\r\n", "2024-10-18 12:45:09,646 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1580.pt\r\n", "2024-10-18 12:45:31,699 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1590.pt\r\n", "2024-10-18 12:45:54,783 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1600.pt\r\n", "2024-10-18 12:46:07,955 INFO [train.py:774] Epoch 1, batch 1600, train_loss[loss=3.369, ArTop10Accuracy=0.8038, NarTop10Accuracy=0.5128, over 1417.00 frames. ], tot_loss[loss=3.277, ArTop10Accuracy=0.7859, NarTop10Accuracy=0.5609, over 1148.74 frames. ], batch size: 4, lr: 4.88e-03\r\n", "2024-10-18 12:46:18,034 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1610.pt\r\n", "2024-10-18 12:46:37,391 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1620.pt\r\n", "2024-10-18 12:46:55,874 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1630.pt\r\n", "2024-10-18 12:47:14,534 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1640.pt\r\n", "2024-10-18 12:47:34,239 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1650.pt\r\n", "2024-10-18 12:47:58,533 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1660.pt\r\n", "2024-10-18 12:48:17,331 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1670.pt\r\n", "2024-10-18 12:48:35,091 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1680.pt\r\n", "2024-10-18 12:48:53,177 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1690.pt\r\n", "2024-10-18 12:49:12,166 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1700.pt\r\n", "2024-10-18 12:49:21,346 INFO [train.py:774] Epoch 1, batch 1700, train_loss[loss=3.178, ArTop10Accuracy=0.7873, NarTop10Accuracy=0.6002, over 1072.00 frames. ], tot_loss[loss=3.275, ArTop10Accuracy=0.7868, NarTop10Accuracy=0.56, over 1150.36 frames. ], batch size: 2, lr: 4.87e-03\r\n", "2024-10-18 12:49:30,173 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1710.pt\r\n", "2024-10-18 12:49:47,957 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1720.pt\r\n", "2024-10-18 12:50:05,787 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1730.pt\r\n", "2024-10-18 12:50:24,988 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1740.pt\r\n", "2024-10-18 12:50:42,569 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1750.pt\r\n", "2024-10-18 12:51:01,572 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1760.pt\r\n", "2024-10-18 12:51:20,385 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1770.pt\r\n", "2024-10-18 12:51:38,385 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1780.pt\r\n", "2024-10-18 12:51:56,183 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1790.pt\r\n", "2024-10-18 12:52:22,902 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1800.pt\r\n", "2024-10-18 12:52:32,110 INFO [train.py:774] Epoch 1, batch 1800, train_loss[loss=3.588, ArTop10Accuracy=0.7692, NarTop10Accuracy=0.4603, over 1252.00 frames. ], tot_loss[loss=3.256, ArTop10Accuracy=0.7894, NarTop10Accuracy=0.5633, over 1140.03 frames. ], batch size: 3, lr: 4.85e-03\r\n", "2024-10-18 12:52:51,981 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1810.pt\r\n", "2024-10-18 12:53:09,856 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1820.pt\r\n", "2024-10-18 12:53:28,248 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1830.pt\r\n", "2024-10-18 12:53:46,955 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1840.pt\r\n", "2024-10-18 12:54:09,462 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1850.pt\r\n", "2024-10-18 12:54:29,080 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1860.pt\r\n", "2024-10-18 12:54:47,545 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1870.pt\r\n", "2024-10-18 12:55:06,285 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1880.pt\r\n", "2024-10-18 12:55:25,300 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1890.pt\r\n", "2024-10-18 12:55:44,141 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1900.pt\r\n", "2024-10-18 12:55:53,118 INFO [train.py:774] Epoch 1, batch 1900, train_loss[loss=3.194, ArTop10Accuracy=0.7953, NarTop10Accuracy=0.5671, over 1158.00 frames. ], tot_loss[loss=3.265, ArTop10Accuracy=0.7882, NarTop10Accuracy=0.5627, over 1149.25 frames. ], batch size: 3, lr: 4.83e-03\r\n", "2024-10-18 12:56:01,665 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1910.pt\r\n", "2024-10-18 12:56:20,011 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1920.pt\r\n", "2024-10-18 12:56:38,949 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1930.pt\r\n", "2024-10-18 12:56:58,441 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1940.pt\r\n", "2024-10-18 12:57:15,890 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1950.pt\r\n", "2024-10-18 12:57:36,778 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1960.pt\r\n", "2024-10-18 12:57:55,757 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1970.pt\r\n", "2024-10-18 12:58:14,450 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1980.pt\r\n", "2024-10-18 12:58:33,023 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-1990.pt\r\n", "2024-10-18 12:58:56,351 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2000.pt\r\n", "2024-10-18 12:59:05,683 INFO [train.py:774] Epoch 1, batch 2000, train_loss[loss=3.038, ArTop10Accuracy=0.8255, NarTop10Accuracy=0.6433, over 1026.00 frames. ], tot_loss[loss=3.276, ArTop10Accuracy=0.7868, NarTop10Accuracy=0.5603, over 1142.00 frames. ], batch size: 4, lr: 4.82e-03\r\n", "2024-10-18 12:59:06,846 INFO [utils.py:877] Clipping_scale=2.0, grad-norm quartiles 3.169e+01 4.378e+01 4.693e+01 5.101e+01 1.046e+02, threshold=9.387e+01, percent-clipped=0.0\r\n", "2024-10-18 12:59:15,965 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2010.pt\r\n", "2024-10-18 12:59:34,668 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2020.pt\r\n", "2024-10-18 12:59:53,774 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2030.pt\r\n", "2024-10-18 13:00:11,941 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2040.pt\r\n", "2024-10-18 13:00:30,647 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2050.pt\r\n", "2024-10-18 13:00:58,554 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2060.pt\r\n", "2024-10-18 13:01:15,869 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2070.pt\r\n", "2024-10-18 13:01:34,697 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2080.pt\r\n", "2024-10-18 13:01:54,146 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2090.pt\r\n", "2024-10-18 13:02:13,045 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2100.pt\r\n", "2024-10-18 13:02:22,904 INFO [train.py:774] Epoch 1, batch 2100, train_loss[loss=2.903, ArTop10Accuracy=0.8456, NarTop10Accuracy=0.6449, over 1263.00 frames. ], tot_loss[loss=3.262, ArTop10Accuracy=0.7912, NarTop10Accuracy=0.5608, over 1147.32 frames. ], batch size: 5, lr: 4.80e-03\r\n", "2024-10-18 13:02:31,912 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2110.pt\r\n", "2024-10-18 13:02:51,217 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2120.pt\r\n", "2024-10-18 13:03:10,959 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2130.pt\r\n", "2024-10-18 13:03:30,630 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2140.pt\r\n", "2024-10-18 13:03:48,488 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2150.pt\r\n", "2024-10-18 13:04:17,759 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2160.pt\r\n", "2024-10-18 13:04:35,003 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2170.pt\r\n", "2024-10-18 13:04:54,375 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2180.pt\r\n", "2024-10-18 13:05:23,778 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2190.pt\r\n", "2024-10-18 13:05:48,918 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2200.pt\r\n", "2024-10-18 13:05:58,233 INFO [train.py:774] Epoch 1, batch 2200, train_loss[loss=3.73, ArTop10Accuracy=0.7279, NarTop10Accuracy=0.4351, over 680.00 frames. ], tot_loss[loss=3.253, ArTop10Accuracy=0.7921, NarTop10Accuracy=0.565, over 1149.62 frames. ], batch size: 1, lr: 4.78e-03\r\n", "2024-10-18 13:06:07,372 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2210.pt\r\n", "2024-10-18 13:06:26,047 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2220.pt\r\n", "2024-10-18 13:06:44,136 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2230.pt\r\n", "2024-10-18 13:07:03,493 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2240.pt\r\n", "2024-10-18 13:07:25,430 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2250.pt\r\n", "2024-10-18 13:07:43,620 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2260.pt\r\n", "2024-10-18 13:08:02,337 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2270.pt\r\n", "2024-10-18 13:08:34,387 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2280.pt\r\n", "2024-10-18 13:08:54,311 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2290.pt\r\n", "2024-10-18 13:09:12,573 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2300.pt\r\n", "2024-10-18 13:09:21,918 INFO [train.py:774] Epoch 1, batch 2300, train_loss[loss=3.244, ArTop10Accuracy=0.7913, NarTop10Accuracy=0.5515, over 1131.00 frames. ], tot_loss[loss=3.256, ArTop10Accuracy=0.7914, NarTop10Accuracy=0.5632, over 1148.13 frames. ], batch size: 3, lr: 4.77e-03\r\n", "2024-10-18 13:09:31,173 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2310.pt\r\n", "2024-10-18 13:09:58,910 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2320.pt\r\n", "2024-10-18 13:10:17,351 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2330.pt\r\n", "2024-10-18 13:10:35,602 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2340.pt\r\n", "2024-10-18 13:10:54,923 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2350.pt\r\n", "2024-10-18 13:11:26,916 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2360.pt\r\n", "2024-10-18 13:11:45,780 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2370.pt\r\n", "2024-10-18 13:12:05,229 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2380.pt\r\n", "2024-10-18 13:12:24,229 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2390.pt\r\n", "2024-10-18 13:12:43,045 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2400.pt\r\n", "2024-10-18 13:12:51,939 INFO [train.py:774] Epoch 1, batch 2400, train_loss[loss=3.139, ArTop10Accuracy=0.8364, NarTop10Accuracy=0.595, over 1400.00 frames. ], tot_loss[loss=3.25, ArTop10Accuracy=0.7914, NarTop10Accuracy=0.566, over 1155.33 frames. ], batch size: 2, lr: 4.75e-03\r\n", "2024-10-18 13:13:01,267 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2410.pt\r\n", "2024-10-18 13:13:21,168 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2420.pt\r\n", "2024-10-18 13:13:39,201 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2430.pt\r\n", "2024-10-18 13:13:58,183 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2440.pt\r\n", "2024-10-18 13:14:19,262 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2450.pt\r\n", "2024-10-18 13:14:49,246 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2460.pt\r\n", "2024-10-18 13:15:07,775 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2470.pt\r\n", "2024-10-18 13:15:39,219 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2480.pt\r\n", "2024-10-18 13:15:58,285 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2490.pt\r\n", "2024-10-18 13:16:16,148 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2500.pt\r\n", "2024-10-18 13:16:25,760 INFO [train.py:774] Epoch 1, batch 2500, train_loss[loss=3.115, ArTop10Accuracy=0.8171, NarTop10Accuracy=0.6034, over 1110.00 frames. ], tot_loss[loss=3.238, ArTop10Accuracy=0.7923, NarTop10Accuracy=0.5691, over 1154.27 frames. ], batch size: 3, lr: 4.73e-03\r\n", "2024-10-18 13:16:35,390 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2510.pt\r\n", "2024-10-18 13:17:04,414 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2520.pt\r\n", "2024-10-18 13:17:23,916 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2530.pt\r\n", "2024-10-18 13:17:42,719 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2540.pt\r\n", "2024-10-18 13:18:01,168 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2550.pt\r\n", "2024-10-18 13:18:20,084 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2560.pt\r\n", "2024-10-18 13:18:37,967 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2570.pt\r\n", "2024-10-18 13:18:55,912 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2580.pt\r\n", "2024-10-18 13:19:14,357 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2590.pt\r\n", "2024-10-18 13:19:33,799 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2600.pt\r\n", "2024-10-18 13:19:43,323 INFO [train.py:774] Epoch 1, batch 2600, train_loss[loss=3.213, ArTop10Accuracy=0.7744, NarTop10Accuracy=0.6219, over 1281.00 frames. ], tot_loss[loss=3.225, ArTop10Accuracy=0.7933, NarTop10Accuracy=0.5713, over 1157.52 frames. ], batch size: 3, lr: 4.71e-03\r\n", "2024-10-18 13:19:52,496 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2610.pt\r\n", "2024-10-18 13:20:11,147 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2620.pt\r\n", "2024-10-18 13:20:29,689 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2630.pt\r\n", "2024-10-18 13:20:48,292 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2640.pt\r\n", "2024-10-18 13:21:11,749 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2650.pt\r\n", "2024-10-18 13:21:31,114 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2660.pt\r\n", "2024-10-18 13:21:49,378 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2670.pt\r\n", "2024-10-18 13:22:08,210 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2680.pt\r\n", "2024-10-18 13:22:27,199 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2690.pt\r\n", "2024-10-18 13:22:54,964 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2700.pt\r\n", "2024-10-18 13:23:04,291 INFO [train.py:774] Epoch 1, batch 2700, train_loss[loss=3.04, ArTop10Accuracy=0.8037, NarTop10Accuracy=0.6468, over 1014.00 frames. ], tot_loss[loss=3.231, ArTop10Accuracy=0.7938, NarTop10Accuracy=0.5693, over 1159.10 frames. ], batch size: 4, lr: 4.69e-03\r\n", "2024-10-18 13:23:13,957 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2710.pt\r\n", "2024-10-18 13:23:38,881 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2720.pt\r\n", "2024-10-18 13:23:58,113 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2730.pt\r\n", "2024-10-18 13:24:27,049 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2740.pt\r\n", "2024-10-18 13:24:45,620 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2750.pt\r\n", "2024-10-18 13:25:04,660 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2760.pt\r\n", "2024-10-18 13:25:24,329 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2770.pt\r\n", "2024-10-18 13:25:54,611 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2780.pt\r\n", "2024-10-18 13:26:12,411 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2790.pt\r\n", "2024-10-18 13:26:31,102 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2800.pt\r\n", "2024-10-18 13:26:40,362 INFO [train.py:774] Epoch 1, batch 2800, train_loss[loss=3.268, ArTop10Accuracy=0.7668, NarTop10Accuracy=0.5751, over 1119.00 frames. ], tot_loss[loss=3.242, ArTop10Accuracy=0.792, NarTop10Accuracy=0.5676, over 1151.06 frames. ], batch size: 2, lr: 4.67e-03\r\n", "2024-10-18 13:26:50,667 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2810.pt\r\n", "2024-10-18 13:27:09,872 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2820.pt\r\n", "2024-10-18 13:27:28,051 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2830.pt\r\n", "2024-10-18 13:27:46,256 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2840.pt\r\n", "2024-10-18 13:28:04,943 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2850.pt\r\n", "2024-10-18 13:28:32,466 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2860.pt\r\n", "2024-10-18 13:28:52,102 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2870.pt\r\n", "2024-10-18 13:29:10,941 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2880.pt\r\n", "2024-10-18 13:29:30,094 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2890.pt\r\n", "2024-10-18 13:29:58,884 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2900.pt\r\n", "2024-10-18 13:30:07,732 INFO [train.py:774] Epoch 1, batch 2900, train_loss[loss=3.357, ArTop10Accuracy=0.8063, NarTop10Accuracy=0.5351, over 1270.00 frames. ], tot_loss[loss=3.246, ArTop10Accuracy=0.7926, NarTop10Accuracy=0.5654, over 1154.25 frames. ], batch size: 3, lr: 4.65e-03\r\n", "2024-10-18 13:30:17,403 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2910.pt\r\n", "2024-10-18 13:30:36,046 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2920.pt\r\n", "2024-10-18 13:31:07,580 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2930.pt\r\n", "2024-10-18 13:31:25,986 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2940.pt\r\n", "2024-10-18 13:31:45,433 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2950.pt\r\n", "2024-10-18 13:32:04,741 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2960.pt\r\n", "2024-10-18 13:32:23,101 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2970.pt\r\n", "2024-10-18 13:32:42,057 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2980.pt\r\n", "2024-10-18 13:33:00,916 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-2990.pt\r\n", "2024-10-18 13:33:19,865 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3000.pt\r\n", "2024-10-18 13:33:29,501 INFO [train.py:774] Epoch 1, batch 3000, train_loss[loss=3.093, ArTop10Accuracy=0.7894, NarTop10Accuracy=0.6209, over 1244.00 frames. ], tot_loss[loss=3.233, ArTop10Accuracy=0.7951, NarTop10Accuracy=0.5686, over 1159.64 frames. ], batch size: 5, lr: 4.63e-03\r\n", "2024-10-18 13:33:30,147 INFO [utils.py:877] Clipping_scale=2.0, grad-norm quartiles 2.737e+01 4.212e+01 4.549e+01 4.879e+01 7.841e+01, threshold=9.097e+01, percent-clipped=0.0\r\n", "2024-10-18 13:33:38,575 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3010.pt\r\n", "2024-10-18 13:33:57,500 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3020.pt\r\n", "2024-10-18 13:34:16,087 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3030.pt\r\n", "2024-10-18 13:34:34,314 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3040.pt\r\n", "2024-10-18 13:34:53,104 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3050.pt\r\n", "2024-10-18 13:35:11,588 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3060.pt\r\n", "2024-10-18 13:35:30,190 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3070.pt\r\n", "2024-10-18 13:35:48,620 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3080.pt\r\n", "2024-10-18 13:36:07,288 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3090.pt\r\n", "2024-10-18 13:36:26,123 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3100.pt\r\n", "2024-10-18 13:36:35,703 INFO [train.py:774] Epoch 1, batch 3100, train_loss[loss=2.971, ArTop10Accuracy=0.8153, NarTop10Accuracy=0.6594, over 1056.00 frames. ], tot_loss[loss=3.227, ArTop10Accuracy=0.7966, NarTop10Accuracy=0.5713, over 1142.86 frames. ], batch size: 2, lr: 4.61e-03\r\n", "2024-10-18 13:36:53,250 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3110.pt\r\n", "2024-10-18 13:37:11,548 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3120.pt\r\n", "2024-10-18 13:37:30,658 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3130.pt\r\n", "2024-10-18 13:37:50,010 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3140.pt\r\n", "2024-10-18 13:38:08,117 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3150.pt\r\n", "2024-10-18 13:38:27,021 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3160.pt\r\n", "2024-10-18 13:38:45,153 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3170.pt\r\n", "2024-10-18 13:39:04,755 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3180.pt\r\n", "2024-10-18 13:39:24,057 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3190.pt\r\n", "2024-10-18 13:39:43,492 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3200.pt\r\n", "2024-10-18 13:39:53,261 INFO [train.py:774] Epoch 1, batch 3200, train_loss[loss=3.034, ArTop10Accuracy=0.8173, NarTop10Accuracy=0.6121, over 1182.00 frames. ], tot_loss[loss=3.238, ArTop10Accuracy=0.7952, NarTop10Accuracy=0.567, over 1155.64 frames. ], batch size: 3, lr: 4.59e-03\r\n", "2024-10-18 13:40:03,281 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3210.pt\r\n", "2024-10-18 13:40:21,725 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3220.pt\r\n", "2024-10-18 13:40:48,689 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3230.pt\r\n", "2024-10-18 13:41:07,026 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3240.pt\r\n", "2024-10-18 13:41:24,841 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3250.pt\r\n", "2024-10-18 13:41:44,181 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3260.pt\r\n", "2024-10-18 13:42:03,304 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3270.pt\r\n", "2024-10-18 13:42:21,201 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3280.pt\r\n", "2024-10-18 13:42:41,163 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3290.pt\r\n", "2024-10-18 13:43:00,758 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3300.pt\r\n", "2024-10-18 13:43:10,263 INFO [train.py:774] Epoch 1, batch 3300, train_loss[loss=3.083, ArTop10Accuracy=0.8263, NarTop10Accuracy=0.5899, over 1376.00 frames. ], tot_loss[loss=3.238, ArTop10Accuracy=0.7953, NarTop10Accuracy=0.5661, over 1160.94 frames. ], batch size: 4, lr: 4.57e-03\r\n", "2024-10-18 13:43:19,697 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3310.pt\r\n", "2024-10-18 13:43:37,766 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3320.pt\r\n", "2024-10-18 13:43:56,600 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3330.pt\r\n", "2024-10-18 13:44:24,096 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3340.pt\r\n", "2024-10-18 13:44:42,692 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3350.pt\r\n", "2024-10-18 13:45:01,596 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3360.pt\r\n", "2024-10-18 13:45:22,523 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3370.pt\r\n", "2024-10-18 13:45:41,567 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3380.pt\r\n", "2024-10-18 13:46:01,227 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3390.pt\r\n", "2024-10-18 13:46:20,337 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3400.pt\r\n", "2024-10-18 13:46:29,902 INFO [train.py:774] Epoch 1, batch 3400, train_loss[loss=3.077, ArTop10Accuracy=0.8108, NarTop10Accuracy=0.5995, over 1205.00 frames. ], tot_loss[loss=3.258, ArTop10Accuracy=0.7953, NarTop10Accuracy=0.5609, over 1161.76 frames. ], batch size: 9, lr: 4.55e-03\r\n", "2024-10-18 13:46:39,570 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3410.pt\r\n", "2024-10-18 13:46:58,262 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3420.pt\r\n", "2024-10-18 13:47:17,155 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3430.pt\r\n", "2024-10-18 13:47:35,020 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3440.pt\r\n", "2024-10-18 13:47:55,306 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3450.pt\r\n", "2024-10-18 13:48:14,574 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3460.pt\r\n", "2024-10-18 13:48:34,352 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3470.pt\r\n", "2024-10-18 13:48:53,096 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3480.pt\r\n", "2024-10-18 13:49:12,124 INFO [utils.py:237] Saving checkpoint to exp/valle_dev/checkpoint-3490.pt\r\n", "2024-10-18 13:49:27,782 INFO [train.py:658] Reaches end of dataloader.\r\n", "Traceback (most recent call last):\r\n", " File \"/kaggle/working/train.py\", line 1078, in \r\n", " main()\r\n", " File \"/kaggle/working/train.py\", line 1071, in main\r\n", " run(rank=0, world_size=1, args=args)\r\n", " File \"/kaggle/working/train.py\", line 1025, in run\r\n", " save_checkpoint(\r\n", " File \"/kaggle/working/train.py\", line 468, in save_checkpoint\r\n", " if params.cur_epoch % params.save_every == 0:\r\n", " File \"/kaggle/working/train_utils/icefall/utils.py\", line 433, in __getattr__\r\n", " raise AttributeError(f\"No such attribute '{key}'\")\r\n", "AttributeError: No such attribute 'save_every'\r\n" ] } ], "source": [ "!python3 train.py \\\n", " --dtype \"bfloat16\" \\\n", " --num-epochs 100 \\\n", " --save-every-n 10 \\\n", " --world-size 1 \\\n", " --keep-last-k 2 \\\n", " --visualize True \\\n", " --train_dir $input_train_path \\\n", " --valid_dir $input_test_path " ] }, { "cell_type": "code", "execution_count": 11, "id": "0676b6cc", "metadata": { "execution": { "iopub.execute_input": "2024-10-18T13:49:34.838834Z", "iopub.status.busy": "2024-10-18T13:49:34.836865Z", "iopub.status.idle": "2024-10-18T13:49:35.990064Z", "shell.execute_reply": "2024-10-18T13:49:35.988873Z" }, "papermill": { "duration": 1.201327, "end_time": "2024-10-18T13:49:35.992594", "exception": false, "start_time": "2024-10-18T13:49:34.791267", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "cp: cannot stat '/kaggle/working/exp/valle_dev/best-train-loss.pt': No such file or directory\r\n" ] } ], "source": [ "!cp /kaggle/working/exp/valle_dev/best-train-loss.pt /checkpoints/vallex-checkpoint.pt" ] }, { "cell_type": "code", "execution_count": 12, "id": "04927ed9", "metadata": { "execution": { "iopub.execute_input": "2024-10-18T13:49:36.081399Z", "iopub.status.busy": "2024-10-18T13:49:36.081003Z", "iopub.status.idle": "2024-10-18T13:50:03.703924Z", "shell.execute_reply": "2024-10-18T13:50:03.703094Z" }, "papermill": { "duration": 27.670293, "end_time": "2024-10-18T13:50:03.706716", "exception": false, "start_time": "2024-10-18T13:49:36.036423", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/kaggle/working/utils/generation.py:78: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n", " checkpoint = torch.load(os.path.join(checkpoints_dir, model_checkpoint_name), map_location='cpu')\n", "/opt/conda/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:134: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.\n", " WeightNorm.apply(module, name, dim)\n", "/opt/conda/lib/python3.10/site-packages/vocos/pretrained.py:70: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n", " state_dict = torch.load(model_path, map_location=\"cpu\")\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "VALL-E EOS [0 -> 102]\n" ] }, { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from utils.generation import SAMPLE_RATE, generate_audio, preload_models\n", "from scipy.io.wavfile import write as write_wav\n", "from IPython.display import Audio\n", "\n", "# download and load all models\n", "preload_models()\n", "\n", "# generate audio from text\n", "text_prompt = \"\"\"\n", "Xin chao, Viet Nam.\n", "\"\"\"\n", "audio_array = generate_audio(text_prompt)\n", "\n", "# save audio to disk\n", "write_wav(\"vallex_generation.wav\", SAMPLE_RATE, audio_array)\n", "\n", "# play text in notebook\n", "Audio(audio_array, rate=SAMPLE_RATE)" ] } ], "metadata": { "kaggle": { "accelerator": "gpu", "dataSources": [ { "datasetId": 5859839, "sourceId": 9652454, "sourceType": "datasetVersion" } ], "dockerImageVersionId": 30787, "isGpuEnabled": true, "isInternetEnabled": true, "language": "python", "sourceType": "notebook" }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" }, "papermill": { "default_parameters": {}, "duration": 7312.253211, "end_time": "2024-10-18T13:50:05.580095", "environment_variables": {}, "exception": null, "input_path": "__notebook__.ipynb", "output_path": "__notebook__.ipynb", "parameters": {}, "start_time": "2024-10-18T11:48:13.326884", "version": "2.6.0" } }, "nbformat": 4, "nbformat_minor": 5 }