From e67c6e4f188c91027d4b9f7f062af5746ba4ede4 Mon Sep 17 00:00:00 2001 From: Stef de Wildt Date: Mon, 8 Jun 2026 14:00:44 +0200 Subject: [PATCH 1/6] Fix inference stability on single GPU Also some job files for reproducing results --- .gitignore | 5 ++++- logs/.gitkeep => .project-root | 0 jobs/create_embeddings.job | 24 +++++++++++++++++++++++ jobs/inference_sid.job | 29 ++++++++++++++++++++++++++++ jobs/install.job | 35 ++++++++++++++++++++++++++++++++++ jobs/tiger_inference.job | 27 ++++++++++++++++++++++++++ jobs/train_sid.job | 29 ++++++++++++++++++++++++++++ jobs/ttrain_tiger.job | 26 +++++++++++++++++++++++++ src/utils/inference_utils.py | 15 ++++++++++++--- 9 files changed, 186 insertions(+), 4 deletions(-) rename logs/.gitkeep => .project-root (100%) mode change 100755 => 100644 create mode 100644 jobs/create_embeddings.job create mode 100644 jobs/inference_sid.job create mode 100644 jobs/install.job create mode 100644 jobs/tiger_inference.job create mode 100644 jobs/train_sid.job create mode 100644 jobs/ttrain_tiger.job diff --git a/.gitignore b/.gitignore index 7b004e5..6c346c4 100644 --- a/.gitignore +++ b/.gitignore @@ -191,4 +191,7 @@ cython_debug/ # exclude from AI features like autocomplete and code analysis. Recommended for sensitive data # refer to https://docs.cursor.com/context/ignore-files .cursorignore -.cursorindexingignore \ No newline at end of file +.cursorindexingignore + +# job files +/jobs/outputs \ No newline at end of file diff --git a/logs/.gitkeep b/.project-root old mode 100755 new mode 100644 similarity index 100% rename from logs/.gitkeep rename to .project-root diff --git a/jobs/create_embeddings.job b/jobs/create_embeddings.job new file mode 100644 index 0000000..75750b9 --- /dev/null +++ b/jobs/create_embeddings.job @@ -0,0 +1,24 @@ +#!/bin/bash +#SBATCH --job-name=embeddings +#SBATCH --partition=gpu_a100 +#SBATCH --gpus=1 +#SBATCH --time=2:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=jobs/outputs/%x-%j.out +#SBATCH --error=jobs/outputs/%x-%j.err + +cd $HOME/GRID + +module purge +module load 2025 +module load Anaconda3/2025.06-1 + +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +python -m src.inference \ + experiment=sem_embeds_inference_flat \ + data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys \ + hydra.run.dir=/projects/prjs2120/groups/group_08/results/embeddings/toys + diff --git a/jobs/inference_sid.job b/jobs/inference_sid.job new file mode 100644 index 0000000..7f52808 --- /dev/null +++ b/jobs/inference_sid.job @@ -0,0 +1,29 @@ +#!/bin/bash +#SBATCH --job-name=inference_sids +#SBATCH --partition=gpu_a100 +#SBATCH --gpus=1 +#SBATCH --time=4:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=logs/%x-%j.out +#SBATCH --error=logs/%x-%j.err + +module purge +module load 2025 +module load Anaconda3/2025.06-1 + +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +cd $HOME/GRID + +BASE=/projects/prjs2120/groups/group_08 + +python -m src.inference experiment=rkmeans_inference_flat \ + data_dir=$BASE/data/amazon_data/beauty \ + embedding_path=$BASE/results/embeddings/beauty/merged_predictions_tensor.pt \ + embedding_dim=2048 \ + num_hierarchies=3 \ + codebook_width=256 \ + ckpt_path=$BASE/results/sid/beauty/rkmeans_train/*/checkpoints/*.ckpt \ + hydra.run.dir=$BASE/results/sid/beauty/rkmeans_inference \ No newline at end of file diff --git a/jobs/install.job b/jobs/install.job new file mode 100644 index 0000000..00db6ff --- /dev/null +++ b/jobs/install.job @@ -0,0 +1,35 @@ +#!/bin/bash +#SBATCH --job-name=recsys_install +#SBATCH --partition=gpu_mig +#SBATCH --gpus=1 +#SBATCH --time=2:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=jobs/outputs/%x-%j.out +#SBATCH --error=jobs/outputs/%x-%j.err + +cd $HOME/FoMo + +module purge +module load 2025 +module load Anaconda3/2025.06-1 +module load CUDA/12.4.1 # Cleanly exposes nvcc to DeepSpeed + +# 2. Initialize Conda base hooks +source "$EBROOTANACONDA3/etc/profile.d/conda.sh" + +# 3. Use conda-forge to bypass Anaconda's commercial Terms of Service lock +conda create -n RecSys python=3.10 -c conda-forge -y +conda activate RecSys + +# 4. Tell DeepSpeed where your cluster CUDA module lives +export CUDA_HOME=$EBROOTCUDA + +# 5. Install PyTorch matching CUDA 12.4 +pip install torch==2.6.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124 + +# 6. Install fbgemm-gpu +pip install fbgemm-gpu==1.1.0+cu124 --index-url https://download.pytorch.org/whl/cu124 + +# 7. Install the repository dependencies file +pip install -r /home/scur1219/GRID/requirements.txt \ No newline at end of file diff --git a/jobs/tiger_inference.job b/jobs/tiger_inference.job new file mode 100644 index 0000000..2bc4694 --- /dev/null +++ b/jobs/tiger_inference.job @@ -0,0 +1,27 @@ +#!/bin/bash +#SBATCH --job-name=tiger_inf_beauty +#SBATCH --partition=gpu_a100 +#SBATCH --gpus=1 +#SBATCH --time=2:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=logs/tiger_inf_%j.out +#SBATCH --error=logs/tiger_inf_%j.err + +module purge +module load 2025 +module load Anaconda3/2025.06-1 + +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +cd $HOME/GRID + +BASE=/projects/prjs2120/groups/group_08 + +python -m src.inference experiment=tiger_inference_flat \ + data_dir=$BASE/data/amazon_data/beauty \ + semantic_id_path=$BASE/results/sid/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt \ + ckpt_path=$BASE/results/tiger/beauty/*/checkpoints/*.ckpt \ + num_hierarchies=4 \ + hydra.run.dir=$BASE/results/tiger/beauty/inference \ No newline at end of file diff --git a/jobs/train_sid.job b/jobs/train_sid.job new file mode 100644 index 0000000..af485c5 --- /dev/null +++ b/jobs/train_sid.job @@ -0,0 +1,29 @@ +#!/bin/bash +#SBATCH --job-name=train_sids +#SBATCH --partition=gpu_a100 +#SBATCH --gpus=1 +#SBATCH --time=4:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=logs/%x-%j.out +#SBATCH --error=logs/%x-%j.err + +module purge +module load 2025 +module load Anaconda3/2025.06-1 + +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +cd $HOME/GRID + +BASE=/projects/prjs2120/groups/group_08 + +python -m src.train experiment=rkmeans_train_flat \ + data_dir=$BASE/data/amazon_data/beauty \ + embedding_path=$BASE/results/embeddings/beauty/merged_predictions_tensor.pt \ + embedding_dim=2048 \ + num_hierarchies=3 \ + codebook_width=256 \ + hydra.run.dir=$BASE/results/sid/beauty/rkmeans_train + diff --git a/jobs/ttrain_tiger.job b/jobs/ttrain_tiger.job new file mode 100644 index 0000000..75dc18c --- /dev/null +++ b/jobs/ttrain_tiger.job @@ -0,0 +1,26 @@ +#!/bin/bash +#SBATCH --job-name=tiger_train_beauty +#SBATCH --partition=gpu_a100 +#SBATCH --gpus=1 +#SBATCH --time=6:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=logs/tiger_train_%j.out +#SBATCH --error=logs/tiger_train_%j.err + +module purge +module load 2025 +module load Anaconda3/2025.06-1 + +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +cd $HOME/GRID + +BASE=/projects/prjs2120/groups/group_08 + +python -m src.train experiment=tiger_train_flat \ + data_dir=$BASE/data/amazon_data/beauty \ + semantic_id_path=$BASE/results/sid/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt \ + num_hierarchies=4 \ + hydra.run.dir=$BASE/results/tiger/beauty \ No newline at end of file diff --git a/src/utils/inference_utils.py b/src/utils/inference_utils.py index cbde1c3..5d43876 100755 --- a/src/utils/inference_utils.py +++ b/src/utils/inference_utils.py @@ -233,14 +233,20 @@ def on_predict_end( if self.should_merge_files_on_main: # if we use multiple workers, we need to wait for all of them to finish writing # before merging the files - if trainer.global_rank != None: + if ( + torch.distributed.is_available() + and torch.distributed.is_initialized() + ): torch.distributed.barrier() if self.global_rank == 0: log.info("Merging pickle files on main process.") self._merge_files() # other processes can continue after merging - if trainer.global_rank != None: + if ( + torch.distributed.is_available() + and torch.distributed.is_initialized() + ): torch.distributed.barrier() # conducting post-processing functions on the files @@ -253,7 +259,10 @@ def on_predict_end( process_func["function"](file_path) else: process_func["function"](file_path) - if trainer.global_rank != None: + if ( + torch.distributed.is_available() + and torch.distributed.is_initialized() + ): torch.distributed.barrier() def _merge_files(self): From f3746fdc5265e84d955ee544e124a43ae5f0fa4e Mon Sep 17 00:00:00 2001 From: Stef de Wildt Date: Mon, 8 Jun 2026 15:25:09 +0200 Subject: [PATCH 2/6] jobs updated --- jobs/create_embeddings.job | 10 +++++++--- jobs/inference_sid.job | 2 ++ jobs/tiger_inference.job | 2 ++ jobs/train_sid.job | 6 ++++-- jobs/{ttrain_tiger.job => train_tiger.job} | 2 ++ 5 files changed, 17 insertions(+), 5 deletions(-) rename jobs/{ttrain_tiger.job => train_tiger.job} (96%) diff --git a/jobs/create_embeddings.job b/jobs/create_embeddings.job index 75750b9..3595c06 100644 --- a/jobs/create_embeddings.job +++ b/jobs/create_embeddings.job @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=embeddings -#SBATCH --partition=gpu_a100 +#SBATCH --partition=gpu_h100 #SBATCH --gpus=1 #SBATCH --time=2:00:00 #SBATCH --cpus-per-task=8 @@ -17,8 +17,12 @@ module load Anaconda3/2025.06-1 source $(conda info --base)/etc/profile.d/conda.sh conda activate RecSys +export CUDA_VISIBLE_DEVICES=0 +export WORLD_SIZE=1 +export OMP_NUM_THREADS=8 + python -m src.inference \ experiment=sem_embeds_inference_flat \ - data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys \ - hydra.run.dir=/projects/prjs2120/groups/group_08/results/embeddings/toys + data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports \ + hydra.run.dir=/projects/prjs2120/groups/group_08/results/embeddings/sports diff --git a/jobs/inference_sid.job b/jobs/inference_sid.job index 7f52808..d3ce9e9 100644 --- a/jobs/inference_sid.job +++ b/jobs/inference_sid.job @@ -17,6 +17,8 @@ conda activate RecSys cd $HOME/GRID +export OMP_NUM_THREADS=8 + BASE=/projects/prjs2120/groups/group_08 python -m src.inference experiment=rkmeans_inference_flat \ diff --git a/jobs/tiger_inference.job b/jobs/tiger_inference.job index 2bc4694..dbcb1e4 100644 --- a/jobs/tiger_inference.job +++ b/jobs/tiger_inference.job @@ -15,6 +15,8 @@ module load Anaconda3/2025.06-1 source $(conda info --base)/etc/profile.d/conda.sh conda activate RecSys +export OMP_NUM_THREADS=8 + cd $HOME/GRID BASE=/projects/prjs2120/groups/group_08 diff --git a/jobs/train_sid.job b/jobs/train_sid.job index af485c5..c1d4081 100644 --- a/jobs/train_sid.job +++ b/jobs/train_sid.job @@ -1,5 +1,5 @@ #!/bin/bash -#SBATCH --job-name=train_sids +#SBATCH --job-name=train_sids_beauty #SBATCH --partition=gpu_a100 #SBATCH --gpus=1 #SBATCH --time=4:00:00 @@ -17,11 +17,13 @@ conda activate RecSys cd $HOME/GRID +export OMP_NUM_THREADS=8 + BASE=/projects/prjs2120/groups/group_08 python -m src.train experiment=rkmeans_train_flat \ data_dir=$BASE/data/amazon_data/beauty \ - embedding_path=$BASE/results/embeddings/beauty/merged_predictions_tensor.pt \ + embedding_path=$BASE/results/embeddings/beauty_v2/merged_predictions_tensor.pt \ embedding_dim=2048 \ num_hierarchies=3 \ codebook_width=256 \ diff --git a/jobs/ttrain_tiger.job b/jobs/train_tiger.job similarity index 96% rename from jobs/ttrain_tiger.job rename to jobs/train_tiger.job index 75dc18c..72a67d2 100644 --- a/jobs/ttrain_tiger.job +++ b/jobs/train_tiger.job @@ -15,6 +15,8 @@ module load Anaconda3/2025.06-1 source $(conda info --base)/etc/profile.d/conda.sh conda activate RecSys +export OMP_NUM_THREADS=8 + cd $HOME/GRID BASE=/projects/prjs2120/groups/group_08 From 341e671916d9491769134e27fd8a7cb490cad2ab Mon Sep 17 00:00:00 2001 From: Stef de Wildt Date: Wed, 10 Jun 2026 18:43:20 +0200 Subject: [PATCH 3/6] Updated jobs >first make metadata folders otherwise error --- .gitignore | 3 ++- jobs/inference_sid.job | 12 ++++++------ jobs/train_sid.job | 6 +++--- jobs/train_tiger.job | 8 ++++---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 6c346c4..f7cabe4 100644 --- a/.gitignore +++ b/.gitignore @@ -194,4 +194,5 @@ cython_debug/ .cursorindexingignore # job files -/jobs/outputs \ No newline at end of file +/jobs/outputs +/logs \ No newline at end of file diff --git a/jobs/inference_sid.job b/jobs/inference_sid.job index d3ce9e9..d136655 100644 --- a/jobs/inference_sid.job +++ b/jobs/inference_sid.job @@ -1,8 +1,8 @@ #!/bin/bash -#SBATCH --job-name=inference_sids +#SBATCH --job-name=inference_sids_toys #SBATCH --partition=gpu_a100 #SBATCH --gpus=1 -#SBATCH --time=4:00:00 +#SBATCH --time=1:00:00 #SBATCH --cpus-per-task=8 #SBATCH --mem=32G #SBATCH --output=logs/%x-%j.out @@ -22,10 +22,10 @@ export OMP_NUM_THREADS=8 BASE=/projects/prjs2120/groups/group_08 python -m src.inference experiment=rkmeans_inference_flat \ - data_dir=$BASE/data/amazon_data/beauty \ - embedding_path=$BASE/results/embeddings/beauty/merged_predictions_tensor.pt \ + data_dir=$BASE/data/amazon_data/toys \ + embedding_path=$BASE/results/embeddings/toys/pickle/merged_predictions_tensor.pt \ embedding_dim=2048 \ num_hierarchies=3 \ codebook_width=256 \ - ckpt_path=$BASE/results/sid/beauty/rkmeans_train/*/checkpoints/*.ckpt \ - hydra.run.dir=$BASE/results/sid/beauty/rkmeans_inference \ No newline at end of file + ckpt_path=$BASE/results/sid_rkmeans/toys/rkmeans_train/checkpoints/checkpoint_000_000030.ckpt \ + hydra.run.dir=$BASE/results/sid_rkmeans/toys/rkmeans_inference \ No newline at end of file diff --git a/jobs/train_sid.job b/jobs/train_sid.job index c1d4081..f28efdf 100644 --- a/jobs/train_sid.job +++ b/jobs/train_sid.job @@ -2,7 +2,7 @@ #SBATCH --job-name=train_sids_beauty #SBATCH --partition=gpu_a100 #SBATCH --gpus=1 -#SBATCH --time=4:00:00 +#SBATCH --time=1:00:00 #SBATCH --cpus-per-task=8 #SBATCH --mem=32G #SBATCH --output=logs/%x-%j.out @@ -23,9 +23,9 @@ BASE=/projects/prjs2120/groups/group_08 python -m src.train experiment=rkmeans_train_flat \ data_dir=$BASE/data/amazon_data/beauty \ - embedding_path=$BASE/results/embeddings/beauty_v2/merged_predictions_tensor.pt \ + embedding_path=$BASE/results/embeddings/beauty_v2/pickle/merged_predictions_tensor.pt \ embedding_dim=2048 \ num_hierarchies=3 \ codebook_width=256 \ - hydra.run.dir=$BASE/results/sid/beauty/rkmeans_train + hydra.run.dir=$BASE/results/sid_rkmeans/beauty/rkmeans_train diff --git a/jobs/train_tiger.job b/jobs/train_tiger.job index 72a67d2..6b78c8e 100644 --- a/jobs/train_tiger.job +++ b/jobs/train_tiger.job @@ -1,5 +1,5 @@ #!/bin/bash -#SBATCH --job-name=tiger_train_beauty +#SBATCH --job-name=tiger_train_sports #SBATCH --partition=gpu_a100 #SBATCH --gpus=1 #SBATCH --time=6:00:00 @@ -22,7 +22,7 @@ cd $HOME/GRID BASE=/projects/prjs2120/groups/group_08 python -m src.train experiment=tiger_train_flat \ - data_dir=$BASE/data/amazon_data/beauty \ - semantic_id_path=$BASE/results/sid/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt \ + data_dir=$BASE/data/amazon_data/sports \ + semantic_id_path=$BASE/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt \ num_hierarchies=4 \ - hydra.run.dir=$BASE/results/tiger/beauty \ No newline at end of file + hydra.run.dir=$BASE/results/tiger/sports_rkmeans \ No newline at end of file From 7b29128a76d3d8c3804e5595b1af086f37ba8476 Mon Sep 17 00:00:00 2001 From: Stef de Wildt Date: Thu, 11 Jun 2026 23:56:41 +0200 Subject: [PATCH 4/6] tried to do table 2 --- jobs/embedding.sh | 57 ++++ jobs/jobs/outputs/embeds-23695733.err | 30 ++ jobs/jobs/outputs/embeds-23695733.out | 474 ++++++++++++++++++++++++++ jobs/rkmean_inference.sh | 48 +++ jobs/rkmeans_train.sh | 46 +++ jobs/tiger_inference.job | 12 +- jobs/tiger_train.sh | 33 ++ 7 files changed, 696 insertions(+), 4 deletions(-) create mode 100644 jobs/embedding.sh create mode 100644 jobs/jobs/outputs/embeds-23695733.err create mode 100644 jobs/jobs/outputs/embeds-23695733.out create mode 100644 jobs/rkmean_inference.sh create mode 100644 jobs/rkmeans_train.sh create mode 100644 jobs/tiger_train.sh diff --git a/jobs/embedding.sh b/jobs/embedding.sh new file mode 100644 index 0000000..d5dd797 --- /dev/null +++ b/jobs/embedding.sh @@ -0,0 +1,57 @@ +#!/bin/bash +#SBATCH --job-name=embeds +#SBATCH --partition=gpu_h100 +#SBATCH --gpus=1 +#SBATCH --time=2:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=jobs/outputs/%x-%j.out +#SBATCH --error=jobs/outputs/%x-%j.err + +# MODEL_SIZE=$1 # Pass L or XXL +# DATASET=$2 # Pass beauty, toys, or sports + +# # Force the input to uppercase just in case +# MODEL_SIZE=$(echo "$1" | tr '[:lower:]' '[:upper:]') +# DATASET=$2 + +# if [[ -z "$MODEL_SIZE" || -z "$DATASET" ]]; then +# echo "Usage: sbatch embedding.sh [L|XXL] [beauty|toys|sports]" +# exit 1 +# fi + +# # Case-insensitive mapping block +# if [[ "$MODEL_SIZE" == "L" || "$MODEL_SIZE" == "LARGE" ]]; then +# HF_MODEL="large" +# elif [[ "$MODEL_SIZE" == "XXL" ]]; then +# HF_MODEL="xxl" +# else +# echo "Invalid model size: $MODEL_SIZE. Choose L or XXL." +# exit 1 +# fi + +cd $HOME/GRID + +module purge +module load 2025 +module load Anaconda3/2025.06-1 +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +export CUDA_VISIBLE_DEVICES=0 +export WORLD_SIZE=1 +export OMP_NUM_THREADS=8 + +BASE=/projects/prjs2120/groups/group_08 + +# python -m src.inference \ +# experiment=sem_embeds_inference_flat \ +# data_dir=$BASE/data/amazon_data/$DATASET \ +# hydra.run.dir=$BASE/results/embeddings/${MODEL_SIZE}/$DATASET \ +# embedding_model=google/flan-t5-${HF_MODEL} + +python -m src.inference \ + experiment=sem_embeds_inference_flat \ + data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty \ + hydra.run.dir=/projects/prjs2120/groups/group_08/results/embeddings/L/beauty \ + embedding_model=google/flan-t5-large \ No newline at end of file diff --git a/jobs/jobs/outputs/embeds-23695733.err b/jobs/jobs/outputs/embeds-23695733.err new file mode 100644 index 0000000..298143c --- /dev/null +++ b/jobs/jobs/outputs/embeds-23695733.err @@ -0,0 +1,30 @@ +============================================================================================== +Warning! Mixing Conda and module environments may lead to corruption of the +user environment. +We do not recommend users mixing those two environments unless absolutely +necessary. Note that +SURF does not provide any support for Conda environment. +For more information, please refer to our software policy page: +https://servicedesk.surf.nl/wiki/display/WIKI/Software+policy+Snellius#SoftwarepolicySnellius-UseofAnacondaandMinicondaenvironmentsonSnellius + +Remember that many packages have already been installed on the system and can +be loaded using +the 'module load ' command. If you are uncertain if a package is +already available +on the system, please use 'module avail' or 'module spider' to search for it. +============================================================================================== +2026-06-11 23:53:06.529256: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-06-11 23:53:06.569158: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +[rank: 0] Seed set to 42 +/home/scur1219/.conda/envs/RecSys/lib/python3.10/site-packages/lightning/fabric/plugins/environments/slurm.py:204: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /gpfs/home6/scur1219/GRID/src/inference.py experimen ... +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +HPU available: False, using: 0 HPUs +You are using a CUDA device ('NVIDIA H100') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] +2026-06-11 23:53:23.322268: I tensorflow/core/kernels/data/tf_record_dataset_op.cc:370] TFRecordDataset `buffer_size` is unspecified, default to 262144 +2026-06-11 23:53:23.327050: I tensorflow/core/kernels/data/tf_record_dataset_op.cc:370] TFRecordDataset `buffer_size` is unspecified, default to 262144 +/home/scur1219/.conda/envs/RecSys/lib/python3.10/site-packages/lightning/pytorch/loops/prediction_loop.py:307: Couldn't infer the batch indices fetched from your dataloader: `DataloaderWithIterationRetry` +2026-06-11 23:54:02.968686: I tensorflow/core/framework/local_rendezvous.cc:405] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence +2026-06-11 23:54:03.617516: I tensorflow/core/framework/local_rendezvous.cc:405] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence diff --git a/jobs/jobs/outputs/embeds-23695733.out b/jobs/jobs/outputs/embeds-23695733.out new file mode 100644 index 0000000..6379bbd --- /dev/null +++ b/jobs/jobs/outputs/embeds-23695733.out @@ -0,0 +1,474 @@ +[2026-06-11 23:53:09,818][src.utils.utils][INFO] - [rank: 0] Enforcing tags!  +[2026-06-11 23:53:09,876][src.utils.utils][WARNING] - [rank: 0] Config loss was not found in the config tree. Make sure this is expected. +[2026-06-11 23:53:09,876][src.utils.utils][WARNING] - [rank: 0] Config optim was not found in the config tree. Make sure this is expected. +[2026-06-11 23:53:09,876][src.utils.utils][WARNING] - [rank: 0] Config eval was not found in the config tree. Make sure this is expected. +[2026-06-11 23:53:12,879][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich!  +CONFIG +├── data_loading +│ └── tokenizer_config: +│ max_length: 128 +│ padding: max_length +│ truncation: true +│ add_special_tokens: true +│ postprocess_eos_token: false +│ tokenizer: +│ _target_: transformers.AutoTokenizer.from_pretrained +│ pretrained_model_name_or_path: google/flan-t5-large +│ features_config: +│ features: +│ - name: id +│ num_placeholder_tokens: 0 +│ is_item_ids: true +│ embeddings: ??? +│ type: +│ _target_: torch.__dict__.get +│ _args_: +│ - int32 +│ - name: text +│ type: +│ _target_: torch.__dict__.get +│ _args_: +│ - bytes +│ is_text: true +│ - name: embedding +│ type: +│ _target_: torch.__dict__.get +│ _args_: +│ - float32 +│ is_embedding: true +│ dataset_config: +│ dataset: +│ _target_: src.data.loading.components.interfaces.ItemDatasetConfig +│ item_id_field: id +│ keep_item_id: true +│ iterate_per_row: true +│ data_iterator: +│ _target_: src.data.loading.components.iterators.TFRecordIterator +│ features_to_consider: +│ - text +│ num_placeholder_tokens_map: +│ id: 0 +│ preprocessing_functions: +│ - _target_: src.data.loading.components.pre_processing.filter_featur +│ _partial_: true +│ - _target_: src.data.loading.components.pre_processing.convert_to_de +│ _partial_: true +│ features_to_apply: +│ - id +│ - text +│ - _target_: src.data.loading.components.pre_processing.convert_field +│ _partial_: true +│ features_to_apply: +│ - id +│ - _target_: src.data.loading.components.pre_processing.convert_bytes +│ _partial_: true +│ features_to_apply: +│ - text +│ - _target_: src.data.loading.components.pre_processing.tokenize_text +│ _partial_: true +│ features_to_apply: +│ - text +│ tokenizer_config: +│ max_length: 128 +│ padding: max_length +│ truncation: true +│ add_special_tokens: true +│ postprocess_eos_token: false +│ tokenizer: +│ _target_: transformers.AutoTokenizer.from_pretrained +│ pretrained_model_name_or_path: google/flan-t5-large +│ - _target_: src.data.loading.components.pre_processing.squeeze_tenso +│ _partial_: true +│ features_to_apply: +│ - text +│ - text_mask +│ field_type_map: +│ id: +│ _target_: torch.__dict__.get +│ _args_: +│ - int32 +│ text: +│ _target_: torch.__dict__.get +│ _args_: +│ - bytes +│ embedding: +│ _target_: torch.__dict__.get +│ _args_: +│ - float32 +│ datamodule: +│ _target_: src.data.loading.datamodules.sequence_datamodule.ItemDataMod +│ predict_dataloader_config: +│ _target_: src.data.loading.components.interfaces.ItemDataloaderConfi +│ dataset_class: +│ _target_: src.data.loading.components.dataloading.UnboundedSequenc +│ _partial_: true +│ data_folder: /projects/prjs2120/groups/group_08/data/amazon_data/bea +│ should_shuffle_rows: false +│ batch_size_per_device: 8 +│ num_workers: 2 +│ assign_files_by_size: true +│ timeout: 60 +│ drop_last: false +│ pin_memory: false +│ persistent_workers: true +│ collate_fn: +│ _target_: src.data.loading.components.collate_functions.collate_fn +│ _partial_: true +│ item_id_field: id +│ feature_to_input_name: +│ id: item_ids +│ text: text_tokens +│ text_mask: text_mask +│ embedding: input_embedding +│ dataset_config: +│ _target_: src.data.loading.components.interfaces.ItemDatasetConfig +│ item_id_field: id +│ keep_item_id: true +│ iterate_per_row: true +│ data_iterator: +│ _target_: src.data.loading.components.iterators.TFRecordIterator +│ features_to_consider: +│ - text +│ num_placeholder_tokens_map: +│ id: 0 +│ preprocessing_functions: +│ - _target_: src.data.loading.components.pre_processing.filter_feat +│ _partial_: true +│ - _target_: src.data.loading.components.pre_processing.convert_to_ +│ _partial_: true +│ features_to_apply: +│ - id +│ - text +│ - _target_: src.data.loading.components.pre_processing.convert_fie +│ _partial_: true +│ features_to_apply: +│ - id +│ - _target_: src.data.loading.components.pre_processing.convert_byt +│ _partial_: true +│ features_to_apply: +│ - text +│ - _target_: src.data.loading.components.pre_processing.tokenize_te +│ _partial_: true +│ features_to_apply: +│ - text +│ tokenizer_config: +│ max_length: 128 +│ padding: max_length +│ truncation: true +│ add_special_tokens: true +│ postprocess_eos_token: false +│ tokenizer: +│ _target_: transformers.AutoTokenizer.from_pretrained +│ pretrained_model_name_or_path: google/flan-t5-large +│ - _target_: src.data.loading.components.pre_processing.squeeze_ten +│ _partial_: true +│ features_to_apply: +│ - text +│ - text_mask +│ field_type_map: +│ id: +│ _target_: torch.__dict__.get +│ _args_: +│ - int32 +│ text: +│ _target_: torch.__dict__.get +│ _args_: +│ - bytes +│ embedding: +│ _target_: torch.__dict__.get +│ _args_: +│ - float32 +│ limit_files: null +│ +├── model +│ └── loss_function: null +│ optimizer: null +│ scheduler: null +│ evaluator: null +│ _target_: src.modules.semantic_embedding_inference_module.SemanticEmbedd +│ semantic_embedding_model_input_map: +│ input_ids: text_tokens +│ attention_mask: text_mask +│ semantic_embedding_model: +│ _target_: src.components.network_blocks.hf_language_model.HFLanguageMo +│ huggingface_model: +│ _target_: transformers.T5EncoderModel.from_pretrained +│ pretrained_model_name_or_path: google/flan-t5-large +│ aggregator: +│ _target_: src.models.components.network_blocks.embedding_aggregator. +│ aggregation_strategy: +│ _target_: src.models.components.network_blocks.aggregation_strateg +│ +├── callbacks +│ └── bq_writer: null +│ pickle_writer: +│ _target_: src.utils.inference_utils.LocalPickleWriter +│ output_dir: /projects/prjs2120/groups/group_08/results/embeddings/L/be +│ flush_frequency: 64 +│ write_interval: batch +│ should_merge_files_on_main: true +│ prediction_key_name: item_id +│ prediction_name: embedding +│ should_merge_list_of_keyed_tensors_to_single_tensor: true +│ +├── logger +│ └── csv: +│ _target_: lightning.pytorch.loggers.csv_logs.CSVLogger +│ save_dir: /projects/prjs2120/groups/group_08/results/embeddings/L/beau +│ name: csv/ +│ prefix: '' +│ +├── trainer +│ └── _target_: lightning.pytorch.trainer.Trainer +│ default_root_dir: /projects/prjs2120/groups/group_08/results/embeddings/ +│ min_steps: 1 +│ max_steps: 80000 +│ max_epochs: 10 +│ accelerator: gpu +│ devices: -1 +│ num_nodes: 1 +│ precision: 32 +│ log_every_n_steps: 2500 +│ val_check_interval: 5000 +│ deterministic: false +│ accumulate_grad_batches: 1 +│ profiler: +│ _target_: lightning.pytorch.profilers.PassThroughProfiler +│ +├── paths +│ └── root_dir: . +│ data_dir: /projects/prjs2120/groups/group_08/data/amazon_data/beauty +│ log_dir: ./logs +│ output_dir: /projects/prjs2120/groups/group_08/results/embeddings/L/beau +│ work_dir: /gpfs/home6/scur1219/GRID +│ profile_dir: /projects/prjs2120/groups/group_08/results/embeddings/L/bea +│ metadata_dir: /projects/prjs2120/groups/group_08/results/embeddings/L/be +│ +├── extras +│ └── ignore_warnings: false +│ enforce_tags: true +│ print_config_warnings: true +│ print_config: true +│ +├── task_name +│ └── inference +├── id +│ └── 2026-06-11/23-53-09 +├── tags +│ └── ['amazon', 'semantic-embeddings-inference'] +├── experiment +│ └── None +├── ckpt_path +│ └── None +├── data_dir +│ └── /projects/prjs2120/groups/group_08/data/amazon_data/beauty +├── embedding_model +│ └── google/flan-t5-large +└── seed + └── 42 +[2026-06-11 23:53:12,973][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule  +[2026-06-11 23:53:14,881][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model  +[2026-06-11 23:53:20,382][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-11 23:53:20,383][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback  +[2026-06-11 23:53:20,564][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-11 23:53:20,564][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger  +[2026-06-11 23:53:20,566][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer  +[2026-06-11 23:53:20,601][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-11 23:53:20,736][__main__][INFO] - [rank: 0] Starting inference! +[2026-06-11 23:53:20,736][__main__][WARNING] - [rank: 0] No ckpt_path was provided. If using a model you trained, this is mandatory. Only leave ckpt_path=None if using a pre-trained model. +[2026-06-11 23:53:21,302][src.utils.inference_utils][INFO] - Rank 0 initialized for inference. + Predicting: | | 0/? [00:00 Date: Thu, 25 Jun 2026 13:53:13 +0200 Subject: [PATCH 5/6] added all logs from reproducing part --- .gitignore | 3 +- jobs/embedding.sh | 57 +-- jobs/jobs/outputs/embeds-23695733.err | 30 -- jobs/jobs/outputs/embeds-23695733.out | 474 ------------------ jobs/logs/reproducing/XXL/sports | 30 ++ jobs/logs/reproducing/XXL/toys | 30 ++ ...mean_inference.sh => rkmeans_inference.sh} | 15 +- jobs/rkmeans_train.sh | 18 +- jobs/tiger_train.sh | 23 +- jobs/train_tiger.job | 7 +- 10 files changed, 130 insertions(+), 557 deletions(-) delete mode 100644 jobs/jobs/outputs/embeds-23695733.err delete mode 100644 jobs/jobs/outputs/embeds-23695733.out create mode 100644 jobs/logs/reproducing/XXL/sports create mode 100644 jobs/logs/reproducing/XXL/toys rename jobs/{rkmean_inference.sh => rkmeans_inference.sh} (59%) diff --git a/.gitignore b/.gitignore index f7cabe4..d99e828 100644 --- a/.gitignore +++ b/.gitignore @@ -195,4 +195,5 @@ cython_debug/ # job files /jobs/outputs -/logs \ No newline at end of file +*.err +*.out \ No newline at end of file diff --git a/jobs/embedding.sh b/jobs/embedding.sh index d5dd797..6083303 100644 --- a/jobs/embedding.sh +++ b/jobs/embedding.sh @@ -8,27 +8,24 @@ #SBATCH --output=jobs/outputs/%x-%j.out #SBATCH --error=jobs/outputs/%x-%j.err -# MODEL_SIZE=$1 # Pass L or XXL -# DATASET=$2 # Pass beauty, toys, or sports - -# # Force the input to uppercase just in case -# MODEL_SIZE=$(echo "$1" | tr '[:lower:]' '[:upper:]') -# DATASET=$2 - -# if [[ -z "$MODEL_SIZE" || -z "$DATASET" ]]; then -# echo "Usage: sbatch embedding.sh [L|XXL] [beauty|toys|sports]" -# exit 1 -# fi - -# # Case-insensitive mapping block -# if [[ "$MODEL_SIZE" == "L" || "$MODEL_SIZE" == "LARGE" ]]; then -# HF_MODEL="large" -# elif [[ "$MODEL_SIZE" == "XXL" ]]; then -# HF_MODEL="xxl" -# else -# echo "Invalid model size: $MODEL_SIZE. Choose L or XXL." -# exit 1 -# fi +MODEL_SIZE_INPUT=$1 # Pass L or XXL +DATASET=$2 # Pass beauty, toys, or sports + +if [[ -z "$MODEL_SIZE_INPUT" || -z "$DATASET" ]]; then + echo "Usage: sbatch embedding.sh [L|XXL] [beauty|toys|sports]" + exit 1 +fi + +MODEL_SIZE=$(echo "$MODEL_SIZE_INPUT" | tr '[:lower:]' '[:upper:]') + +if [[ "$MODEL_SIZE" == "L" || "$MODEL_SIZE" == "LARGE" ]]; then + HF_MODEL="large" +elif [[ "$MODEL_SIZE" == "XXL" ]]; then + HF_MODEL="xxl" +else + echo "Invalid model size: $MODEL_SIZE_INPUT. Choose L or XXL." + exit 1 +fi cd $HOME/GRID @@ -38,20 +35,20 @@ module load Anaconda3/2025.06-1 source $(conda info --base)/etc/profile.d/conda.sh conda activate RecSys +# --- FORCE ALL STORAGE & TEMP WRITES TO SCRATCH --- +export HF_HOME=/scratch-shared/$USER/hf_cache +export TMPDIR=/scratch-shared/$USER/tmp +mkdir -p $HF_HOME +mkdir -p $TMPDIR + export CUDA_VISIBLE_DEVICES=0 export WORLD_SIZE=1 export OMP_NUM_THREADS=8 BASE=/projects/prjs2120/groups/group_08 -# python -m src.inference \ -# experiment=sem_embeds_inference_flat \ -# data_dir=$BASE/data/amazon_data/$DATASET \ -# hydra.run.dir=$BASE/results/embeddings/${MODEL_SIZE}/$DATASET \ -# embedding_model=google/flan-t5-${HF_MODEL} - python -m src.inference \ experiment=sem_embeds_inference_flat \ - data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty \ - hydra.run.dir=/projects/prjs2120/groups/group_08/results/embeddings/L/beauty \ - embedding_model=google/flan-t5-large \ No newline at end of file + data_dir=$BASE/data/amazon_data/$DATASET \ + hydra.run.dir=/scratch-shared/$USER/embeddings/${MODEL_SIZE}/$DATASET \ + embedding_model=google/flan-t5-${HF_MODEL} \ No newline at end of file diff --git a/jobs/jobs/outputs/embeds-23695733.err b/jobs/jobs/outputs/embeds-23695733.err deleted file mode 100644 index 298143c..0000000 --- a/jobs/jobs/outputs/embeds-23695733.err +++ /dev/null @@ -1,30 +0,0 @@ -============================================================================================== -Warning! Mixing Conda and module environments may lead to corruption of the -user environment. -We do not recommend users mixing those two environments unless absolutely -necessary. Note that -SURF does not provide any support for Conda environment. -For more information, please refer to our software policy page: -https://servicedesk.surf.nl/wiki/display/WIKI/Software+policy+Snellius#SoftwarepolicySnellius-UseofAnacondaandMinicondaenvironmentsonSnellius - -Remember that many packages have already been installed on the system and can -be loaded using -the 'module load ' command. If you are uncertain if a package is -already available -on the system, please use 'module avail' or 'module spider' to search for it. -============================================================================================== -2026-06-11 23:53:06.529256: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. -2026-06-11 23:53:06.569158: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. -To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. -[rank: 0] Seed set to 42 -/home/scur1219/.conda/envs/RecSys/lib/python3.10/site-packages/lightning/fabric/plugins/environments/slurm.py:204: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /gpfs/home6/scur1219/GRID/src/inference.py experimen ... -GPU available: True (cuda), used: True -TPU available: False, using: 0 TPU cores -HPU available: False, using: 0 HPUs -You are using a CUDA device ('NVIDIA H100') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision -LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] -2026-06-11 23:53:23.322268: I tensorflow/core/kernels/data/tf_record_dataset_op.cc:370] TFRecordDataset `buffer_size` is unspecified, default to 262144 -2026-06-11 23:53:23.327050: I tensorflow/core/kernels/data/tf_record_dataset_op.cc:370] TFRecordDataset `buffer_size` is unspecified, default to 262144 -/home/scur1219/.conda/envs/RecSys/lib/python3.10/site-packages/lightning/pytorch/loops/prediction_loop.py:307: Couldn't infer the batch indices fetched from your dataloader: `DataloaderWithIterationRetry` -2026-06-11 23:54:02.968686: I tensorflow/core/framework/local_rendezvous.cc:405] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence -2026-06-11 23:54:03.617516: I tensorflow/core/framework/local_rendezvous.cc:405] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence diff --git a/jobs/jobs/outputs/embeds-23695733.out b/jobs/jobs/outputs/embeds-23695733.out deleted file mode 100644 index 6379bbd..0000000 --- a/jobs/jobs/outputs/embeds-23695733.out +++ /dev/null @@ -1,474 +0,0 @@ -[2026-06-11 23:53:09,818][src.utils.utils][INFO] - [rank: 0] Enforcing tags!  -[2026-06-11 23:53:09,876][src.utils.utils][WARNING] - [rank: 0] Config loss was not found in the config tree. Make sure this is expected. -[2026-06-11 23:53:09,876][src.utils.utils][WARNING] - [rank: 0] Config optim was not found in the config tree. Make sure this is expected. -[2026-06-11 23:53:09,876][src.utils.utils][WARNING] - [rank: 0] Config eval was not found in the config tree. Make sure this is expected. -[2026-06-11 23:53:12,879][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich!  -CONFIG -├── data_loading -│ └── tokenizer_config: -│ max_length: 128 -│ padding: max_length -│ truncation: true -│ add_special_tokens: true -│ postprocess_eos_token: false -│ tokenizer: -│ _target_: transformers.AutoTokenizer.from_pretrained -│ pretrained_model_name_or_path: google/flan-t5-large -│ features_config: -│ features: -│ - name: id -│ num_placeholder_tokens: 0 -│ is_item_ids: true -│ embeddings: ??? -│ type: -│ _target_: torch.__dict__.get -│ _args_: -│ - int32 -│ - name: text -│ type: -│ _target_: torch.__dict__.get -│ _args_: -│ - bytes -│ is_text: true -│ - name: embedding -│ type: -│ _target_: torch.__dict__.get -│ _args_: -│ - float32 -│ is_embedding: true -│ dataset_config: -│ dataset: -│ _target_: src.data.loading.components.interfaces.ItemDatasetConfig -│ item_id_field: id -│ keep_item_id: true -│ iterate_per_row: true -│ data_iterator: -│ _target_: src.data.loading.components.iterators.TFRecordIterator -│ features_to_consider: -│ - text -│ num_placeholder_tokens_map: -│ id: 0 -│ preprocessing_functions: -│ - _target_: src.data.loading.components.pre_processing.filter_featur -│ _partial_: true -│ - _target_: src.data.loading.components.pre_processing.convert_to_de -│ _partial_: true -│ features_to_apply: -│ - id -│ - text -│ - _target_: src.data.loading.components.pre_processing.convert_field -│ _partial_: true -│ features_to_apply: -│ - id -│ - _target_: src.data.loading.components.pre_processing.convert_bytes -│ _partial_: true -│ features_to_apply: -│ - text -│ - _target_: src.data.loading.components.pre_processing.tokenize_text -│ _partial_: true -│ features_to_apply: -│ - text -│ tokenizer_config: -│ max_length: 128 -│ padding: max_length -│ truncation: true -│ add_special_tokens: true -│ postprocess_eos_token: false -│ tokenizer: -│ _target_: transformers.AutoTokenizer.from_pretrained -│ pretrained_model_name_or_path: google/flan-t5-large -│ - _target_: src.data.loading.components.pre_processing.squeeze_tenso -│ _partial_: true -│ features_to_apply: -│ - text -│ - text_mask -│ field_type_map: -│ id: -│ _target_: torch.__dict__.get -│ _args_: -│ - int32 -│ text: -│ _target_: torch.__dict__.get -│ _args_: -│ - bytes -│ embedding: -│ _target_: torch.__dict__.get -│ _args_: -│ - float32 -│ datamodule: -│ _target_: src.data.loading.datamodules.sequence_datamodule.ItemDataMod -│ predict_dataloader_config: -│ _target_: src.data.loading.components.interfaces.ItemDataloaderConfi -│ dataset_class: -│ _target_: src.data.loading.components.dataloading.UnboundedSequenc -│ _partial_: true -│ data_folder: /projects/prjs2120/groups/group_08/data/amazon_data/bea -│ should_shuffle_rows: false -│ batch_size_per_device: 8 -│ num_workers: 2 -│ assign_files_by_size: true -│ timeout: 60 -│ drop_last: false -│ pin_memory: false -│ persistent_workers: true -│ collate_fn: -│ _target_: src.data.loading.components.collate_functions.collate_fn -│ _partial_: true -│ item_id_field: id -│ feature_to_input_name: -│ id: item_ids -│ text: text_tokens -│ text_mask: text_mask -│ embedding: input_embedding -│ dataset_config: -│ _target_: src.data.loading.components.interfaces.ItemDatasetConfig -│ item_id_field: id -│ keep_item_id: true -│ iterate_per_row: true -│ data_iterator: -│ _target_: src.data.loading.components.iterators.TFRecordIterator -│ features_to_consider: -│ - text -│ num_placeholder_tokens_map: -│ id: 0 -│ preprocessing_functions: -│ - _target_: src.data.loading.components.pre_processing.filter_feat -│ _partial_: true -│ - _target_: src.data.loading.components.pre_processing.convert_to_ -│ _partial_: true -│ features_to_apply: -│ - id -│ - text -│ - _target_: src.data.loading.components.pre_processing.convert_fie -│ _partial_: true -│ features_to_apply: -│ - id -│ - _target_: src.data.loading.components.pre_processing.convert_byt -│ _partial_: true -│ features_to_apply: -│ - text -│ - _target_: src.data.loading.components.pre_processing.tokenize_te -│ _partial_: true -│ features_to_apply: -│ - text -│ tokenizer_config: -│ max_length: 128 -│ padding: max_length -│ truncation: true -│ add_special_tokens: true -│ postprocess_eos_token: false -│ tokenizer: -│ _target_: transformers.AutoTokenizer.from_pretrained -│ pretrained_model_name_or_path: google/flan-t5-large -│ - _target_: src.data.loading.components.pre_processing.squeeze_ten -│ _partial_: true -│ features_to_apply: -│ - text -│ - text_mask -│ field_type_map: -│ id: -│ _target_: torch.__dict__.get -│ _args_: -│ - int32 -│ text: -│ _target_: torch.__dict__.get -│ _args_: -│ - bytes -│ embedding: -│ _target_: torch.__dict__.get -│ _args_: -│ - float32 -│ limit_files: null -│ -├── model -│ └── loss_function: null -│ optimizer: null -│ scheduler: null -│ evaluator: null -│ _target_: src.modules.semantic_embedding_inference_module.SemanticEmbedd -│ semantic_embedding_model_input_map: -│ input_ids: text_tokens -│ attention_mask: text_mask -│ semantic_embedding_model: -│ _target_: src.components.network_blocks.hf_language_model.HFLanguageMo -│ huggingface_model: -│ _target_: transformers.T5EncoderModel.from_pretrained -│ pretrained_model_name_or_path: google/flan-t5-large -│ aggregator: -│ _target_: src.models.components.network_blocks.embedding_aggregator. -│ aggregation_strategy: -│ _target_: src.models.components.network_blocks.aggregation_strateg -│ -├── callbacks -│ └── bq_writer: null -│ pickle_writer: -│ _target_: src.utils.inference_utils.LocalPickleWriter -│ output_dir: /projects/prjs2120/groups/group_08/results/embeddings/L/be -│ flush_frequency: 64 -│ write_interval: batch -│ should_merge_files_on_main: true -│ prediction_key_name: item_id -│ prediction_name: embedding -│ should_merge_list_of_keyed_tensors_to_single_tensor: true -│ -├── logger -│ └── csv: -│ _target_: lightning.pytorch.loggers.csv_logs.CSVLogger -│ save_dir: /projects/prjs2120/groups/group_08/results/embeddings/L/beau -│ name: csv/ -│ prefix: '' -│ -├── trainer -│ └── _target_: lightning.pytorch.trainer.Trainer -│ default_root_dir: /projects/prjs2120/groups/group_08/results/embeddings/ -│ min_steps: 1 -│ max_steps: 80000 -│ max_epochs: 10 -│ accelerator: gpu -│ devices: -1 -│ num_nodes: 1 -│ precision: 32 -│ log_every_n_steps: 2500 -│ val_check_interval: 5000 -│ deterministic: false -│ accumulate_grad_batches: 1 -│ profiler: -│ _target_: lightning.pytorch.profilers.PassThroughProfiler -│ -├── paths -│ └── root_dir: . -│ data_dir: /projects/prjs2120/groups/group_08/data/amazon_data/beauty -│ log_dir: ./logs -│ output_dir: /projects/prjs2120/groups/group_08/results/embeddings/L/beau -│ work_dir: /gpfs/home6/scur1219/GRID -│ profile_dir: /projects/prjs2120/groups/group_08/results/embeddings/L/bea -│ metadata_dir: /projects/prjs2120/groups/group_08/results/embeddings/L/be -│ -├── extras -│ └── ignore_warnings: false -│ enforce_tags: true -│ print_config_warnings: true -│ print_config: true -│ -├── task_name -│ └── inference -├── id -│ └── 2026-06-11/23-53-09 -├── tags -│ └── ['amazon', 'semantic-embeddings-inference'] -├── experiment -│ └── None -├── ckpt_path -│ └── None -├── data_dir -│ └── /projects/prjs2120/groups/group_08/data/amazon_data/beauty -├── embedding_model -│ └── google/flan-t5-large -└── seed - └── 42 -[2026-06-11 23:53:12,973][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule  -[2026-06-11 23:53:14,881][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model  -[2026-06-11 23:53:20,382][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... -[2026-06-11 23:53:20,383][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback  -[2026-06-11 23:53:20,564][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... -[2026-06-11 23:53:20,564][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger  -[2026-06-11 23:53:20,566][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer  -[2026-06-11 23:53:20,601][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! -[2026-06-11 23:53:20,736][__main__][INFO] - [rank: 0] Starting inference! -[2026-06-11 23:53:20,736][__main__][WARNING] - [rank: 0] No ckpt_path was provided. If using a model you trained, this is mandatory. Only leave ckpt_path=None if using a pre-trained model. -[2026-06-11 23:53:21,302][src.utils.inference_utils][INFO] - Rank 0 initialized for inference. - Predicting: | | 0/? [00:00 +[2026-06-16 11:57:34,220][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:34,496][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata +[2026-06-16 11:57:34,497][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans ++should_skip_retry=True +[2026-06-16 11:57:42,317][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:42,322][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:42,596][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-16 11:57:42,606][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-16 11:57:42,704][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-16 11:57:42,951][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-16 11:57:42,951][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,954][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,957][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,957][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,960][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,960][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-16 11:57:42,960][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.960440', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-16 11:57:42,960][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-16 11:57:42,961][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-16 11:57:42,963][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-16 11:57:42,964][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,966][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,967][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:43,013][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-16 11:57:43,419][__main__][INFO] - [rank: 0] Starting training! +[2026-06-16 11:57:45,247][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.960440', 'restarts': [], 'current_run': 0, 'used_ports': ['33033'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-16 17:26:28,230][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-16 17:29:47,164][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/XXL/sports_rkmeans/checkpoints/checkpoint_epoch=000_step=004100.ckpt +[2026-06-16 17:29:47,174][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(9.1292), 'train/loss_step': tensor(7.8743), 'val/loss': tensor(9.1621), 'val/ndcg@5': tensor(0.0202), 'val/ndcg@10': tensor(0.0250), 'val/recall@5': tensor(0.0298), 'val/recall@10': tensor(0.0447), 'train/loss_epoch': tensor(9.1292), 'test/loss': tensor(9.4324), 'test/ndcg@5': tensor(0.0153), 'test/ndcg@10': tensor(0.0197), 'test/recall@5': tensor(0.0230), 'test/recall@10': tensor(0.0367)} +[2026-06-16 17:29:53,575][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/logs/reproducing/XXL/toys b/jobs/logs/reproducing/XXL/toys new file mode 100644 index 0000000..5133d91 --- /dev/null +++ b/jobs/logs/reproducing/XXL/toys @@ -0,0 +1,30 @@ +[2026-06-16 11:57:34,222][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:34,228][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:34,503][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata +[2026-06-16 11:57:34,503][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans ++should_skip_retry=True +[2026-06-16 11:57:42,319][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:42,324][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:42,597][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-16 11:57:42,606][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-16 11:57:42,708][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-16 11:57:42,952][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-16 11:57:42,952][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,955][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,958][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,958][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,961][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,961][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-16 11:57:42,961][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.961393', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans', '++should_skip_retry=True']} +[2026-06-16 11:57:42,961][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-16 11:57:42,962][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-16 11:57:42,963][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-16 11:57:42,965][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,967][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,967][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:43,013][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-16 11:57:43,419][__main__][INFO] - [rank: 0] Starting training! +[2026-06-16 11:57:45,258][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.961393', 'restarts': [], 'current_run': 0, 'used_ports': ['33453'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans', '++should_skip_retry=True']} +[2026-06-16 16:17:12,668][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-16 16:19:01,920][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/XXL/toys_rkmeans/checkpoints/checkpoint_epoch=000_step=004300.ckpt +[2026-06-16 16:19:01,945][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(7.3486), 'train/loss_step': tensor(6.5392), 'val/loss': tensor(9.9951), 'val/ndcg@5': tensor(0.0367), 'val/ndcg@10': tensor(0.0440), 'val/recall@5': tensor(0.0536), 'val/recall@10': tensor(0.0764), 'train/loss_epoch': tensor(7.3486), 'test/loss': tensor(10.2606), 'test/ndcg@5': tensor(0.0264), 'test/ndcg@10': tensor(0.0333), 'test/recall@5': tensor(0.0409), 'test/recall@10': tensor(0.0624)} +[2026-06-16 16:19:09,630][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/rkmean_inference.sh b/jobs/rkmeans_inference.sh similarity index 59% rename from jobs/rkmean_inference.sh rename to jobs/rkmeans_inference.sh index 9d23e7b..c2452ef 100644 --- a/jobs/rkmean_inference.sh +++ b/jobs/rkmeans_inference.sh @@ -34,15 +34,18 @@ conda activate RecSys cd $HOME/GRID export OMP_NUM_THREADS=8 -BASE=/projects/prjs2120/groups/group_08 +BASE_PROJECT=/projects/prjs2120/groups/group_08 +BASE_SCRATCH=/scratch-shared/$USER -mkdir -p $BASE/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_inference/metadata +# Keep metadata/logging directory in project folder (very lightweight text files) +mkdir -p $BASE_PROJECT/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_inference/metadata +# Run inference utilizing completely isolated scratch paths python -m src.inference experiment=rkmeans_inference_flat \ - data_dir=$BASE/data/amazon_data/$DATASET \ - embedding_path=$BASE/results/embeddings/${MODEL_SIZE}/${DATASET}/pickle/merged_predictions_tensor.pt \ + data_dir=$BASE_PROJECT/data/amazon_data/$DATASET \ + embedding_path=$BASE_SCRATCH/embeddings/${MODEL_SIZE}/${DATASET}/pickle/merged_predictions_tensor.pt \ embedding_dim=$DIM \ num_hierarchies=3 \ codebook_width=256 \ - ckpt_path=$BASE/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_train/checkpoints/$CKPT_NAME \ - hydra.run.dir=$BASE/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_inference \ No newline at end of file + ckpt_path=$BASE_SCRATCH/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_train/checkpoints/$CKPT_NAME \ + hydra.run.dir=$BASE_SCRATCH/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_inference \ No newline at end of file diff --git a/jobs/rkmeans_train.sh b/jobs/rkmeans_train.sh index b8985fe..0ef2acc 100644 --- a/jobs/rkmeans_train.sh +++ b/jobs/rkmeans_train.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=train_sids -#SBATCH --partition=gpu_a100 +#SBATCH --partition=gpu_h100 #SBATCH --gpus=1 #SBATCH --time=1:00:00 #SBATCH --cpus-per-task=8 @@ -33,14 +33,20 @@ conda activate RecSys cd $HOME/GRID export OMP_NUM_THREADS=8 -BASE=/projects/prjs2120/groups/group_08 +BASE_PROJECT=/projects/prjs2120/groups/group_08 +BASE_SCRATCH=/scratch-shared/$USER -mkdir -p $BASE/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_train/metadata +# Point directly to your scratch embeddings +EMBED_PATH=$BASE_SCRATCH/embeddings/${MODEL_SIZE}/${DATASET}/pickle/merged_predictions_tensor.pt +# Ensure the log/metadata structure exists in the project folder (lightweight text files) +mkdir -p $BASE_PROJECT/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_train/metadata + +# Force the heavy checkpoint outputs into your scratch playground python -m src.train experiment=rkmeans_train_flat \ - data_dir=$BASE/data/amazon_data/$DATASET \ - embedding_path=$BASE/results/embeddings/${MODEL_SIZE}/${DATASET}/pickle/merged_predictions_tensor.pt \ + data_dir=$BASE_PROJECT/data/amazon_data/$DATASET \ + embedding_path=$EMBED_PATH \ embedding_dim=$DIM \ num_hierarchies=3 \ codebook_width=256 \ - hydra.run.dir=$BASE/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_train \ No newline at end of file + hydra.run.dir=$BASE_SCRATCH/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_train \ No newline at end of file diff --git a/jobs/tiger_train.sh b/jobs/tiger_train.sh index 5f89138..2659fbc 100644 --- a/jobs/tiger_train.sh +++ b/jobs/tiger_train.sh @@ -8,14 +8,17 @@ #SBATCH --output=logs/tiger_train_%j.out #SBATCH --error=logs/tiger_train_%j.err -MODEL_SIZE=$1 -DATASET=$2 +MODEL_SIZE_INPUT=$1 # Pass L or XXL +DATASET=$2 # Pass beauty, toys, or sports -if [[ -z "$MODEL_SIZE" || -z "$DATASET" ]]; then +if [[ -z "$MODEL_SIZE_INPUT" || -z "$DATASET" ]]; then echo "Usage: sbatch tiger_train.sh [L|XXL] [beauty|toys|sports]" exit 1 fi +# Standardise case handling just like your other scripts +MODEL_SIZE=$(echo "$MODEL_SIZE_INPUT" | tr '[:lower:]' '[:upper:]') + module purge module load 2025 module load Anaconda3/2025.06-1 @@ -24,10 +27,16 @@ conda activate RecSys export OMP_NUM_THREADS=8 cd $HOME/GRID -BASE=/projects/prjs2120/groups/group_08 +BASE_PROJECT=/projects/prjs2120/groups/group_08 +BASE_SCRATCH=/scratch-shared/$USER + +# Point semantic_id_path to where rkmeans_inference saved the outputs on scratch +SEMANTIC_ID_PATH=$BASE_SCRATCH/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_inference/pickle/merged_predictions_tensor.pt + +# Run the final generative recommendation training completely on scratch python -m src.train experiment=tiger_train_flat \ - data_dir=$BASE/data/amazon_data/$DATASET \ - semantic_id_path=$BASE/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_inference/pickle/merged_predictions_tensor.pt \ + data_dir=$BASE_PROJECT/data/amazon_data/$DATASET \ + semantic_id_path=$SEMANTIC_ID_PATH \ num_hierarchies=4 \ - hydra.run.dir=$BASE/results/tiger/${MODEL_SIZE}/${DATASET}_rkmeans \ No newline at end of file + hydra.run.dir=$BASE_SCRATCH/results/tiger/${MODEL_SIZE}/${DATASET}_rkmeans \ No newline at end of file diff --git a/jobs/train_tiger.job b/jobs/train_tiger.job index 6b78c8e..3ea7f5f 100644 --- a/jobs/train_tiger.job +++ b/jobs/train_tiger.job @@ -1,8 +1,8 @@ #!/bin/bash #SBATCH --job-name=tiger_train_sports -#SBATCH --partition=gpu_a100 +#SBATCH --partition=gpu_h100 #SBATCH --gpus=1 -#SBATCH --time=6:00:00 +#SBATCH --time=10:00:00 #SBATCH --cpus-per-task=8 #SBATCH --mem=32G #SBATCH --output=logs/tiger_train_%j.out @@ -20,9 +20,10 @@ export OMP_NUM_THREADS=8 cd $HOME/GRID BASE=/projects/prjs2120/groups/group_08 +BASE_SCRATCH=/scratch-shared/$USER python -m src.train experiment=tiger_train_flat \ data_dir=$BASE/data/amazon_data/sports \ semantic_id_path=$BASE/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt \ num_hierarchies=4 \ - hydra.run.dir=$BASE/results/tiger/sports_rkmeans \ No newline at end of file + hydra.run.dir=$BASE_SCRATCH/results/tiger_default_2/sports_rkmeans \ No newline at end of file From d4fe27b1ee98f2dc2bccf6312735ff380a006196 Mon Sep 17 00:00:00 2001 From: Stef de Wildt Date: Thu, 25 Jun 2026 14:02:09 +0200 Subject: [PATCH 6/6] Add logs --- .gitignore | 2 +- jobs/logs/reproducing/L/beauty/train.log | 30 ++++++ jobs/logs/reproducing/L/sports/train.log | 30 ++++++ jobs/logs/reproducing/L/toys/train.log | 30 ++++++ jobs/logs/reproducing/XXL/beauty/train.log | 30 ++++++ .../XXL/{sports => sports/train.log} | 0 .../reproducing/XXL/{toys => toys/train.log} | 0 jobs/logs/reproducing/beauty/train.log | 30 ++++++ jobs/logs/reproducing/sports/train.log | 97 +++++++++++++++++++ jobs/logs/reproducing/toys/train.log | 30 ++++++ 10 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 jobs/logs/reproducing/L/beauty/train.log create mode 100644 jobs/logs/reproducing/L/sports/train.log create mode 100644 jobs/logs/reproducing/L/toys/train.log create mode 100644 jobs/logs/reproducing/XXL/beauty/train.log rename jobs/logs/reproducing/XXL/{sports => sports/train.log} (100%) rename jobs/logs/reproducing/XXL/{toys => toys/train.log} (100%) create mode 100644 jobs/logs/reproducing/beauty/train.log create mode 100644 jobs/logs/reproducing/sports/train.log create mode 100644 jobs/logs/reproducing/toys/train.log diff --git a/.gitignore b/.gitignore index d99e828..c76f1ff 100644 --- a/.gitignore +++ b/.gitignore @@ -56,7 +56,7 @@ cover/ *.pot # Django stuff: -*.log +# *.log local_settings.py db.sqlite3 db.sqlite3-journal diff --git a/jobs/logs/reproducing/L/beauty/train.log b/jobs/logs/reproducing/L/beauty/train.log new file mode 100644 index 0000000..876a9fe --- /dev/null +++ b/jobs/logs/reproducing/L/beauty/train.log @@ -0,0 +1,30 @@ +[2026-06-16 11:57:33,719][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:33,725][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:34,008][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata +[2026-06-16 11:57:34,008][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/beauty_rkmeans ++should_skip_retry=True +[2026-06-16 11:57:41,866][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:41,871][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:42,145][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-16 11:57:42,153][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-16 11:57:42,240][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-16 11:57:42,491][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-16 11:57:42,492][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,495][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,498][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,498][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,500][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,501][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-16 11:57:42,501][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.501360', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/beauty_rkmeans', '++should_skip_retry=True']} +[2026-06-16 11:57:42,501][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-16 11:57:42,502][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-16 11:57:42,503][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-16 11:57:42,505][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,507][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,508][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,554][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-16 11:57:42,950][__main__][INFO] - [rank: 0] Starting training! +[2026-06-16 11:57:44,675][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.501360', 'restarts': [], 'current_run': 0, 'used_ports': ['42809'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/beauty_rkmeans', '++should_skip_retry=True']} +[2026-06-16 17:02:05,020][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-16 17:04:12,741][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/L/beauty_rkmeans/checkpoints/checkpoint_epoch=000_step=004800.ckpt +[2026-06-16 17:04:12,767][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(7.5031), 'train/loss_step': tensor(4.6689), 'val/loss': tensor(9.5383), 'val/ndcg@5': tensor(0.0393), 'val/ndcg@10': tensor(0.0470), 'val/recall@5': tensor(0.0569), 'val/recall@10': tensor(0.0809), 'train/loss_epoch': tensor(7.5031), 'test/loss': tensor(9.8078), 'test/ndcg@5': tensor(0.0304), 'test/ndcg@10': tensor(0.0365), 'test/recall@5': tensor(0.0449), 'test/recall@10': tensor(0.0640)} +[2026-06-16 17:04:21,722][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/logs/reproducing/L/sports/train.log b/jobs/logs/reproducing/L/sports/train.log new file mode 100644 index 0000000..ec6750c --- /dev/null +++ b/jobs/logs/reproducing/L/sports/train.log @@ -0,0 +1,30 @@ +[2026-06-16 11:57:35,632][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:35,642][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:35,922][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata +[2026-06-16 11:57:35,922][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/sports_rkmeans ++should_skip_retry=True +[2026-06-16 11:57:44,389][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:44,394][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:44,670][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-16 11:57:44,679][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-16 11:57:44,801][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-16 11:57:45,077][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-16 11:57:45,077][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,081][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,083][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,084][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,086][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,086][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-16 11:57:45,086][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:45.086844', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-16 11:57:45,087][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-16 11:57:45,087][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-16 11:57:45,089][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-16 11:57:45,091][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,093][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,093][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,141][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-16 11:57:45,571][__main__][INFO] - [rank: 0] Starting training! +[2026-06-16 11:57:47,411][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:45.086844', 'restarts': [], 'current_run': 0, 'used_ports': ['47843'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-16 17:28:49,970][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-16 17:32:11,383][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/L/sports_rkmeans/checkpoints/checkpoint_epoch=000_step=004100.ckpt +[2026-06-16 17:32:11,391][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(9.3024), 'train/loss_step': tensor(8.0710), 'val/loss': tensor(9.2528), 'val/ndcg@5': tensor(0.0193), 'val/ndcg@10': tensor(0.0241), 'val/recall@5': tensor(0.0294), 'val/recall@10': tensor(0.0440), 'train/loss_epoch': tensor(9.3024), 'test/loss': tensor(9.5419), 'test/ndcg@5': tensor(0.0147), 'test/ndcg@10': tensor(0.0182), 'test/recall@5': tensor(0.0223), 'test/recall@10': tensor(0.0331)} +[2026-06-16 17:32:15,236][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/logs/reproducing/L/toys/train.log b/jobs/logs/reproducing/L/toys/train.log new file mode 100644 index 0000000..124493a --- /dev/null +++ b/jobs/logs/reproducing/L/toys/train.log @@ -0,0 +1,30 @@ +[2026-06-16 11:57:50,782][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:50,792][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:51,076][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata +[2026-06-16 11:57:51,077][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/toys_rkmeans ++should_skip_retry=True +[2026-06-16 11:57:59,361][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:59,367][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:59,639][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-16 11:57:59,656][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-16 11:57:59,809][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-16 11:58:00,096][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-16 11:58:00,096][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:58:00,100][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:58:00,102][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:58:00,103][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:58:00,105][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:58:00,106][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-16 11:58:00,106][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:58:00.106169', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/toys_rkmeans', '++should_skip_retry=True']} +[2026-06-16 11:58:00,107][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-16 11:58:00,107][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-16 11:58:00,109][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-16 11:58:00,111][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:58:00,113][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:58:00,113][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:58:00,171][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-16 11:58:01,130][__main__][INFO] - [rank: 0] Starting training! +[2026-06-16 11:58:02,558][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:58:00.106169', 'restarts': [], 'current_run': 0, 'used_ports': ['36927'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/toys_rkmeans', '++should_skip_retry=True']} +[2026-06-16 15:27:48,918][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-16 15:29:38,187][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/L/toys_rkmeans/checkpoints/checkpoint_epoch=000_step=003300.ckpt +[2026-06-16 15:29:38,235][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(7.8262), 'train/loss_step': tensor(6.8433), 'val/loss': tensor(9.8689), 'val/ndcg@5': tensor(0.0340), 'val/ndcg@10': tensor(0.0418), 'val/recall@5': tensor(0.0499), 'val/recall@10': tensor(0.0740), 'train/loss_epoch': tensor(7.8262), 'test/loss': tensor(9.9868), 'test/ndcg@5': tensor(0.0256), 'test/ndcg@10': tensor(0.0322), 'test/recall@5': tensor(0.0400), 'test/recall@10': tensor(0.0607)} +[2026-06-16 15:29:48,727][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/logs/reproducing/XXL/beauty/train.log b/jobs/logs/reproducing/XXL/beauty/train.log new file mode 100644 index 0000000..b347359 --- /dev/null +++ b/jobs/logs/reproducing/XXL/beauty/train.log @@ -0,0 +1,30 @@ +[2026-06-16 11:57:35,636][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:35,642][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:35,921][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata +[2026-06-16 11:57:35,921][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans ++should_skip_retry=True +[2026-06-16 11:57:44,410][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:44,415][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:44,690][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-16 11:57:44,693][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-16 11:57:44,801][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-16 11:57:45,076][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-16 11:57:45,076][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,080][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,082][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,083][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,085][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,085][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-16 11:57:45,085][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:45.085780', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans', '++should_skip_retry=True']} +[2026-06-16 11:57:45,086][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-16 11:57:45,086][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-16 11:57:45,088][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-16 11:57:45,090][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,092][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,092][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,141][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-16 11:57:45,571][__main__][INFO] - [rank: 0] Starting training! +[2026-06-16 11:57:47,361][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:45.085780', 'restarts': [], 'current_run': 0, 'used_ports': ['58781'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans', '++should_skip_retry=True']} +[2026-06-16 15:51:22,627][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-16 15:53:25,977][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/XXL/beauty_rkmeans/checkpoints/checkpoint_epoch=000_step=003500.ckpt +[2026-06-16 15:53:26,005][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(7.6635), 'train/loss_step': tensor(7.8335), 'val/loss': tensor(9.0442), 'val/ndcg@5': tensor(0.0382), 'val/ndcg@10': tensor(0.0467), 'val/recall@5': tensor(0.0575), 'val/recall@10': tensor(0.0838), 'train/loss_epoch': tensor(7.6635), 'test/loss': tensor(9.2698), 'test/ndcg@5': tensor(0.0262), 'test/ndcg@10': tensor(0.0334), 'test/recall@5': tensor(0.0404), 'test/recall@10': tensor(0.0628)} +[2026-06-16 15:53:34,482][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/logs/reproducing/XXL/sports b/jobs/logs/reproducing/XXL/sports/train.log similarity index 100% rename from jobs/logs/reproducing/XXL/sports rename to jobs/logs/reproducing/XXL/sports/train.log diff --git a/jobs/logs/reproducing/XXL/toys b/jobs/logs/reproducing/XXL/toys/train.log similarity index 100% rename from jobs/logs/reproducing/XXL/toys rename to jobs/logs/reproducing/XXL/toys/train.log diff --git a/jobs/logs/reproducing/beauty/train.log b/jobs/logs/reproducing/beauty/train.log new file mode 100644 index 0000000..470b42e --- /dev/null +++ b/jobs/logs/reproducing/beauty/train.log @@ -0,0 +1,30 @@ +[2026-06-10 10:08:01,933][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-10 10:08:01,940][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-10 10:08:02,223][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata +[2026-06-10 10:08:02,223][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/beauty ++should_skip_retry=True +[2026-06-10 10:08:10,587][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-10 10:08:10,593][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-10 10:08:10,868][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-10 10:08:10,877][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-10 10:08:10,988][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-10 10:08:11,271][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-10 10:08:11,271][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 10:08:11,275][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 10:08:11,277][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 10:08:11,278][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 10:08:11,280][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json +[2026-06-10 10:08:11,281][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-10 10:08:11,281][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json. {'start_time': '2026-06-10T10:08:11.281334', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/beauty', '++should_skip_retry=True']} +[2026-06-10 10:08:11,323][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-10 10:08:11,323][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-10 10:08:11,325][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-10 10:08:11,328][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json +[2026-06-10 10:08:11,331][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json +[2026-06-10 10:08:11,331][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json +[2026-06-10 10:08:11,381][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-10 10:08:11,803][__main__][INFO] - [rank: 0] Starting training! +[2026-06-10 10:08:14,827][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json. {'start_time': '2026-06-10T10:08:11.281334', 'restarts': [], 'current_run': 0, 'used_ports': ['44417'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/beauty', '++should_skip_retry=True']} +[2026-06-10 15:04:16,480][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-10 15:06:30,519][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/work5/0/prjs2120/groups/group_08/results/tiger/beauty/checkpoints/checkpoint_epoch=000_step=004500.ckpt +[2026-06-10 15:06:30,642][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(7.4643), 'train/loss_step': tensor(7.2329), 'val/loss': tensor(9.4356), 'val/ndcg@5': tensor(0.0389), 'val/ndcg@10': tensor(0.0466), 'val/recall@5': tensor(0.0571), 'val/recall@10': tensor(0.0809), 'train/loss_epoch': tensor(7.4643), 'test/loss': tensor(9.6659), 'test/ndcg@5': tensor(0.0304), 'test/ndcg@10': tensor(0.0369), 'test/recall@5': tensor(0.0452), 'test/recall@10': tensor(0.0655)} +[2026-06-10 15:06:39,095][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/logs/reproducing/sports/train.log b/jobs/logs/reproducing/sports/train.log new file mode 100644 index 0000000..7638350 --- /dev/null +++ b/jobs/logs/reproducing/sports/train.log @@ -0,0 +1,97 @@ +[2026-06-10 17:54:07,168][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-10 17:54:07,175][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-10 17:54:07,457][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata +[2026-06-10 17:54:07,457][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans ++should_skip_retry=True +[2026-06-10 17:54:15,436][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-10 17:54:15,443][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-10 17:54:15,715][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-10 17:54:15,725][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-10 17:54:16,001][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-10 17:54:16,296][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-10 17:54:16,297][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:16,300][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:16,302][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:16,303][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:16,305][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:16,306][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-10 17:54:16,306][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:16.306267', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-10 17:54:16,376][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-10 17:54:16,376][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-10 17:54:16,378][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-10 17:54:16,381][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:16,383][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:16,383][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:16,433][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-10 17:54:17,069][__main__][INFO] - [rank: 0] Starting training! +[2026-06-10 17:54:19,682][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:16.306267', 'restarts': [], 'current_run': 0, 'used_ports': ['60229'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-10 23:13:50,592][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-10 23:17:05,964][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/work5/0/prjs2120/groups/group_08/results/tiger/sports_rkmeans/checkpoints/checkpoint_epoch=000_step=004000.ckpt +[2026-06-10 23:17:05,993][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(9.2376), 'train/loss_step': tensor(7.9633), 'val/loss': tensor(9.2298), 'val/ndcg@5': tensor(0.0195), 'val/ndcg@10': tensor(0.0242), 'val/recall@5': tensor(0.0296), 'val/recall@10': tensor(0.0443), 'train/loss_epoch': tensor(9.2376), 'test/loss': tensor(9.5222), 'test/ndcg@5': tensor(0.0153), 'test/ndcg@10': tensor(0.0190), 'test/recall@5': tensor(0.0231), 'test/recall@10': tensor(0.0346)} +[2026-06-10 23:17:11,438][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. +[2026-06-11 11:03:32,516][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-11 11:03:32,544][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-11 11:03:32,823][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata +[2026-06-11 11:03:32,824][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1256/.conda/envs/grid/bin/python /gpfs/home4/scur1256/GRID/src/train.py experiment=tiger_train_flat_dec_only data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans ++should_skip_retry=True +[2026-06-11 11:03:40,640][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-11 11:03:40,646][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-11 11:03:40,918][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-11 11:03:40,928][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-11 11:03:41,226][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-11 11:03:47,839][src.utils.restart_job][ERROR] - [rank: 0] Job failed with return code 1 +[2026-06-11 11:03:47,839][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:03:47,884][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:03:47,884][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:03:47,884][src.utils.restart_job][ERROR] - [rank: 0] Metadata run count 0 does not match current run count 1.This can happen if the exception does not trigger the callback on_exception. Cleaning up andupdating metadata before retrying. +[2026-06-11 11:03:47,900][src.utils.restart_job_utils][INFO] - [rank: 0] Clearing CUDA cache +[2026-06-11 11:03:47,900][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:03:47,901][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:03:47,901][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:16.306267', 'restarts': [], 'current_run': 1, 'used_ports': ['60229'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-11 11:03:47,901][src.utils.restart_job][INFO] - [rank: 0] Retrying in 5 seconds. Attempt 2/4 +[2026-06-11 11:03:52,907][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 2/4): /home/scur1256/.conda/envs/grid/bin/python /gpfs/home4/scur1256/GRID/src/train.py experiment=tiger_train_flat_dec_only data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans ++should_skip_retry=True +[2026-06-11 11:04:00,632][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-11 11:04:00,637][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-11 11:04:00,910][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-11 11:04:00,913][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-11 11:04:00,973][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-11 11:04:02,918][src.utils.restart_job][ERROR] - [rank: 0] Job failed with return code 1 +[2026-06-11 11:04:02,918][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:02,918][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:02,919][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 1 from metadata /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:02,919][src.utils.restart_job][ERROR] - [rank: 0] Metadata run count 1 does not match current run count 2.This can happen if the exception does not trigger the callback on_exception. Cleaning up andupdating metadata before retrying. +[2026-06-11 11:04:02,919][src.utils.restart_job_utils][INFO] - [rank: 0] Clearing CUDA cache +[2026-06-11 11:04:02,919][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:02,920][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:02,920][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:16.306267', 'restarts': [], 'current_run': 2, 'used_ports': ['60229'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-11 11:04:02,920][src.utils.restart_job][INFO] - [rank: 0] Retrying in 5 seconds. Attempt 3/4 +[2026-06-11 11:04:07,925][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 3/4): /home/scur1256/.conda/envs/grid/bin/python /gpfs/home4/scur1256/GRID/src/train.py experiment=tiger_train_flat_dec_only data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans ++should_skip_retry=True +[2026-06-11 11:04:15,545][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-11 11:04:15,550][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-11 11:04:15,824][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-11 11:04:15,830][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-11 11:04:15,889][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-11 11:04:17,936][src.utils.restart_job][ERROR] - [rank: 0] Job failed with return code 1 +[2026-06-11 11:04:17,937][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:17,937][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:17,937][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 2 from metadata /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:17,937][src.utils.restart_job][ERROR] - [rank: 0] Metadata run count 2 does not match current run count 3.This can happen if the exception does not trigger the callback on_exception. Cleaning up andupdating metadata before retrying. +[2026-06-11 11:04:17,938][src.utils.restart_job_utils][INFO] - [rank: 0] Clearing CUDA cache +[2026-06-11 11:04:17,938][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:17,938][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:17,938][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:16.306267', 'restarts': [], 'current_run': 3, 'used_ports': ['60229'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-11 11:04:17,941][src.utils.restart_job][INFO] - [rank: 0] Retrying in 5 seconds. Attempt 4/4 +[2026-06-11 11:04:22,947][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 4/4): /home/scur1256/.conda/envs/grid/bin/python /gpfs/home4/scur1256/GRID/src/train.py experiment=tiger_train_flat_dec_only data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans ++should_skip_retry=True +[2026-06-11 11:04:30,730][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-11 11:04:30,735][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-11 11:04:31,009][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-11 11:04:31,012][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-11 11:04:31,073][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-11 11:04:32,958][src.utils.restart_job][ERROR] - [rank: 0] Job failed with return code 1 +[2026-06-11 11:04:32,958][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:32,958][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:32,959][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 3 from metadata /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:32,959][src.utils.restart_job][ERROR] - [rank: 0] Metadata run count 3 does not match current run count 4.This can happen if the exception does not trigger the callback on_exception. Cleaning up andupdating metadata before retrying. +[2026-06-11 11:04:32,959][src.utils.restart_job_utils][INFO] - [rank: 0] Clearing CUDA cache +[2026-06-11 11:04:32,959][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:32,959][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:32,960][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:16.306267', 'restarts': [], 'current_run': 4, 'used_ports': ['60229'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-11 11:04:32,960][src.utils.restart_job][ERROR] - [rank: 0] Job failed after 3 retries. diff --git a/jobs/logs/reproducing/toys/train.log b/jobs/logs/reproducing/toys/train.log new file mode 100644 index 0000000..8a36678 --- /dev/null +++ b/jobs/logs/reproducing/toys/train.log @@ -0,0 +1,30 @@ +[2026-06-10 17:54:19,912][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-10 17:54:19,921][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-10 17:54:20,206][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata +[2026-06-10 17:54:20,206][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans ++should_skip_retry=True +[2026-06-10 17:54:28,398][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-10 17:54:28,406][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-10 17:54:28,681][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-10 17:54:28,710][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-10 17:54:28,947][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-10 17:54:29,250][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-10 17:54:29,250][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:29,255][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:29,257][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:29,258][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:29,261][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:29,261][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-10 17:54:29,262][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:29.262053', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans', '++should_skip_retry=True']} +[2026-06-10 17:54:29,263][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-10 17:54:29,264][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-10 17:54:29,266][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-10 17:54:29,268][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:29,271][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:29,271][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:29,328][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-10 17:54:30,476][__main__][INFO] - [rank: 0] Starting training! +[2026-06-10 17:54:32,696][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:29.262053', 'restarts': [], 'current_run': 0, 'used_ports': ['48187'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans', '++should_skip_retry=True']} +[2026-06-10 21:00:04,420][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-10 21:01:53,743][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/work5/0/prjs2120/groups/group_08/results/tiger/toys_rkmeans/checkpoints/checkpoint_epoch=000_step=002800.ckpt +[2026-06-10 21:01:53,791][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(7.9583), 'train/loss_step': tensor(6.7817), 'val/loss': tensor(9.4167), 'val/ndcg@5': tensor(0.0345), 'val/ndcg@10': tensor(0.0423), 'val/recall@5': tensor(0.0525), 'val/recall@10': tensor(0.0769), 'train/loss_epoch': tensor(7.9583), 'test/loss': tensor(9.6428), 'test/ndcg@5': tensor(0.0265), 'test/ndcg@10': tensor(0.0329), 'test/recall@5': tensor(0.0410), 'test/recall@10': tensor(0.0605)} +[2026-06-10 21:02:06,365][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully.