Training in progress, step 3000

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81d24d08c866fb7b5e1840e34b47f21cd66ddcbade0fe2b8b2b08e487e34435b
+oid sha256:95d2e59438f2ea8ed38310a3fd0662fe92301766d325f9d8d14a35c37221a4f7
 size 967102601

run.log

@@ -316,3 +316,56 @@ xpu_backend=None,
 {'loss': 0.0003, 'learning_rate': 6.786666666666667e-06, 'epoch': 41.0}
 {'loss': 0.0003, 'learning_rate': 6.731111111111111e-06, 'epoch': 42.0}
 {'loss': 0.0003, 'learning_rate': 6.675555555555556e-06, 'epoch': 42.01}
+{'eval_loss': 0.695440948009491, 'eval_wer': 36.11375535459305, 'eval_runtime': 776.4908, 'eval_samples_per_second': 2.92, 'eval_steps_per_second': 0.091, 'epoch': 42.01}
+{'loss': 0.0003, 'learning_rate': 6.620000000000001e-06, 'epoch': 43.0}
+{'loss': 0.0003, 'learning_rate': 6.564444444444446e-06, 'epoch': 43.01}
+{'loss': 0.0003, 'learning_rate': 6.508888888888889e-06, 'epoch': 44.0}
+{'loss': 0.0003, 'learning_rate': 6.453333333333334e-06, 'epoch': 44.01}
+{'loss': 0.0003, 'learning_rate': 6.397777777777778e-06, 'epoch': 45.0}
+{'loss': 0.0003, 'learning_rate': 6.342222222222223e-06, 'epoch': 45.01}
+{'loss': 0.0003, 'learning_rate': 6.286666666666668e-06, 'epoch': 46.0}
+{'loss': 0.0002, 'learning_rate': 6.231111111111111e-06, 'epoch': 46.01}
+{'loss': 0.0002, 'learning_rate': 6.175555555555556e-06, 'epoch': 47.0}
+{'loss': 0.0002, 'learning_rate': 6.120000000000001e-06, 'epoch': 47.01}
+{'loss': 0.0002, 'learning_rate': 6.064444444444445e-06, 'epoch': 48.0}
+{'loss': 0.0002, 'learning_rate': 6.00888888888889e-06, 'epoch': 48.01}
+{'loss': 0.0002, 'learning_rate': 5.9533333333333345e-06, 'epoch': 49.0}
+{'loss': 0.0002, 'learning_rate': 5.897777777777778e-06, 'epoch': 49.01}
+{'loss': 0.0002, 'learning_rate': 5.8422222222222226e-06, 'epoch': 50.01}
+{'loss': 0.0002, 'learning_rate': 5.7866666666666674e-06, 'epoch': 51.0}
+{'loss': 0.0002, 'learning_rate': 5.7311111111111115e-06, 'epoch': 51.01}
+{'loss': 0.0002, 'learning_rate': 5.675555555555556e-06, 'epoch': 52.0}
+{'loss': 0.0002, 'learning_rate': 5.620000000000001e-06, 'epoch': 52.01}
+{'loss': 0.0002, 'learning_rate': 5.5644444444444444e-06, 'epoch': 53.0}
+{'loss': 0.0002, 'learning_rate': 5.508888888888889e-06, 'epoch': 53.01}
+{'loss': 0.0002, 'learning_rate': 5.453333333333334e-06, 'epoch': 54.0}
+{'loss': 0.0002, 'learning_rate': 5.397777777777778e-06, 'epoch': 54.01}
+{'loss': 0.0002, 'learning_rate': 5.342222222222223e-06, 'epoch': 55.0}
+{'loss': 0.0002, 'learning_rate': 5.286666666666666e-06, 'epoch': 55.01}
+{'loss': 0.0002, 'learning_rate': 5.231111111111111e-06, 'epoch': 56.0}
+{'loss': 0.0002, 'learning_rate': 5.175555555555556e-06, 'epoch': 56.01}
+{'loss': 0.0002, 'learning_rate': 5.12e-06, 'epoch': 57.0}
+{'loss': 0.0002, 'learning_rate': 5.064444444444445e-06, 'epoch': 57.01}
+{'loss': 0.0002, 'learning_rate': 5.00888888888889e-06, 'epoch': 58.0}
+{'loss': 0.0002, 'learning_rate': 4.953333333333334e-06, 'epoch': 59.0}
+{'loss': 0.0002, 'learning_rate': 4.897777777777778e-06, 'epoch': 59.01}
+{'loss': 0.0002, 'learning_rate': 4.842222222222223e-06, 'epoch': 60.0}
+{'loss': 0.0002, 'learning_rate': 4.786666666666667e-06, 'epoch': 60.01}
+{'loss': 0.0002, 'learning_rate': 4.731111111111112e-06, 'epoch': 61.0}
+{'loss': 0.0002, 'learning_rate': 4.675555555555556e-06, 'epoch': 61.01}
+{'loss': 0.0002, 'learning_rate': 4.620000000000001e-06, 'epoch': 62.0}
+{'loss': 0.0002, 'learning_rate': 4.564444444444445e-06, 'epoch': 62.01}
+{'loss': 0.0002, 'learning_rate': 4.50888888888889e-06, 'epoch': 63.0}
+{'loss': 0.0002, 'learning_rate': 4.453333333333334e-06, 'epoch': 63.01}
+12/15/2022 18:02:08 - WARNING - datasets.download.streaming_download_manager - Got disconnected from remote data host. Retrying in 5sec [1/20]
+12/15/2022 18:02:23 - WARNING - datasets.download.streaming_download_manager - Got disconnected from remote data host. Retrying in 5sec [2/20]
+12/15/2022 18:02:38 - WARNING - datasets.download.streaming_download_manager - Got disconnected from remote data host. Retrying in 5sec [3/20]
+12/15/2022 18:02:53 - WARNING - datasets.download.streaming_download_manager - Got disconnected from remote data host. Retrying in 5sec [4/20]
+12/15/2022 18:03:08 - WARNING - datasets.download.streaming_download_manager - Got disconnected from remote data host. Retrying in 5sec [5/20]
+12/15/2022 18:03:23 - WARNING - datasets.download.streaming_download_manager - Got disconnected from remote data host. Retrying in 5sec [6/20]
+12/15/2022 18:03:38 - WARNING - datasets.download.streaming_download_manager - Got disconnected from remote data host. Retrying in 5sec [7/20]
+12/15/2022 18:03:53 - WARNING - datasets.download.streaming_download_manager - Got disconnected from remote data host. Retrying in 5sec [8/20]
+12/15/2022 18:04:08 - WARNING - datasets.download.streaming_download_manager - Got disconnected from remote data host. Retrying in 5sec [9/20]
+12/15/2022 18:04:23 - WARNING - datasets.download.streaming_download_manager - Got disconnected from remote data host. Retrying in 5sec [10/20]
+12/15/2022 18:04:38 - WARNING - datasets.download.streaming_download_manager - Got disconnected from remote data host. Retrying in 5sec [11/20]
+12/15/2022 18:04:43 - WARNING - datasets.download.streaming_download_manager - Got disconnected from remote data host. Retrying in 5sec [12/20]

runs/Dec15_10-25-46_129-146-123-136/events.out.tfevents.1671099977.129-146-123-136.161490.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e507469acc073f07508740e51ae52ada71ff6f2a5c8b2f4226ebfdc35df0637
-size 17455
+oid sha256:3438dee93afb72b3b4950291c95555f51676494ca0da49bd10b700282c7a898d
+size 24053