progress overview

main.py

@@ -309,7 +309,7 @@ def setup_parser():
     #out.add_argument('--dataset', default='/scratch/grzonkow/train_dataset_latest.pkl', type=str, help="Path to dataset")
     #out.add_argument('--safe_dataset', default='C:/Users/cassi/OneDrive/Desktop/Master_Thesis/train_dataset_latest.pkl',
     # type=str, help="Path to safe dataset")
-    out.add_argument('--safe_dataset', default='/scratch/grzonkow/train_dataset_latest.pkl', type=str,
+    out.add_argument('--safe_dataset', default='/scratch/grzonkow/train_dataset_1702.pkl', type=str,
                      help="Path to safe dataset")
     #out.add_argument('--folder', default=r'C:/Users/cassi/OneDrive/Desktop/Master_Thesis/test_file/',
     #                 type=str, help="Path to folder containing packages with music and sm files")
@@ -317,7 +317,7 @@ def setup_parser():
                      type=str, help="Path to folder containing packages with music and sm files")
     #out.add_argument('--model_params_path', default=r'C:/Users/cassi/OneDrive/Desktop/Master_Thesis/model_params.pt',
     #                 type=str, help="Path to safe model parameters")
-    out.add_argument('--model_params_path', default=r'/scratch/grzonkow/model_params.pt',
+    out.add_argument('--model_params_path', default=r'/scratch/grzonkow/model_params_1702.pt',
                      type=str, help="Path to safe model parameters")
     #out.add_argument('--safe_dir', default=None, type=str, help="Path to dataset")
     #out.add_argument('--safe_dir', default="C:/Users/cassi/OneDrive/Desktop/Master_Thesis/output",
@@ -449,6 +449,8 @@ if __name__ == '__main__':
 
     print(f'Length train loader: {len(train_loader)}')
     epochs = 3
+    iter_num = 0
+    num_iters = len(train_loader)
 
     for epoch in range(1, epochs + 1):
         epoch_start_time = time.time()
@@ -456,6 +458,7 @@ if __name__ == '__main__':
         # Run training
         # for each song, get 3 spectrograms and process them through cnn, mlp (difficulty) and transformer
         for imgs, diffs, charts in train_loader:
+            print(f'Progress: {iter_num}/{num_iters}')
             #print(f"Imgs:{imgs}")
             #print("Load data --------------------------------------------")
             #print(f"Imgs shape:{imgs.size()}")
@@ -474,6 +477,7 @@ if __name__ == '__main__':
                   danceformer.scheduler,
                   (pre_conv_imgs, preprocess_difficulty, charts), danceformer.ntokens,
                   danceformer.bptt, epoch, model_params_path, device)
+            iter_num += 1
 
         # val_loss = evaluate(trans_model, val_data)
         # val_ppl = math.exp(val_loss)

model_run.sbatch

@@ -9,8 +9,8 @@
 #SBATCH --partition=informatik-mind
 #SBATCH --gpus=1                         # request 1 GPU
 #SBATCH --gpu_cmode=shared               # Set the GPU into shared mode, so that multiple processes can run on it
-#SBATCH --output=/scratch/grzonkow/model_1602.txt         # capture output
-#SBATCH --error=/scratch/grzonkow/err_1602.txt          # and error streams
+#SBATCH --output=/scratch/grzonkow/model_1702.txt         # capture output
+#SBATCH --error=/scratch/grzonkow/err_1702.txt          # and error streams