diff --git a/.gitignore b/.gitignore index 7b004e5..c76f1ff 100644 --- a/.gitignore +++ b/.gitignore @@ -56,7 +56,7 @@ cover/ *.pot # Django stuff: -*.log +# *.log local_settings.py db.sqlite3 db.sqlite3-journal @@ -191,4 +191,9 @@ cython_debug/ # exclude from AI features like autocomplete and code analysis. Recommended for sensitive data # refer to https://docs.cursor.com/context/ignore-files .cursorignore -.cursorindexingignore \ No newline at end of file +.cursorindexingignore + +# job files +/jobs/outputs +*.err +*.out \ No newline at end of file diff --git a/logs/.gitkeep b/.project-root old mode 100755 new mode 100644 similarity index 100% rename from logs/.gitkeep rename to .project-root diff --git a/jobs/create_embeddings.job b/jobs/create_embeddings.job new file mode 100644 index 0000000..3595c06 --- /dev/null +++ b/jobs/create_embeddings.job @@ -0,0 +1,28 @@ +#!/bin/bash +#SBATCH --job-name=embeddings +#SBATCH --partition=gpu_h100 +#SBATCH --gpus=1 +#SBATCH --time=2:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=jobs/outputs/%x-%j.out +#SBATCH --error=jobs/outputs/%x-%j.err + +cd $HOME/GRID + +module purge +module load 2025 +module load Anaconda3/2025.06-1 + +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +export CUDA_VISIBLE_DEVICES=0 +export WORLD_SIZE=1 +export OMP_NUM_THREADS=8 + +python -m src.inference \ + experiment=sem_embeds_inference_flat \ + data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports \ + hydra.run.dir=/projects/prjs2120/groups/group_08/results/embeddings/sports + diff --git a/jobs/embedding.sh b/jobs/embedding.sh new file mode 100644 index 0000000..6083303 --- /dev/null +++ b/jobs/embedding.sh @@ -0,0 +1,54 @@ +#!/bin/bash +#SBATCH --job-name=embeds +#SBATCH --partition=gpu_h100 +#SBATCH --gpus=1 +#SBATCH --time=2:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=jobs/outputs/%x-%j.out +#SBATCH --error=jobs/outputs/%x-%j.err + +MODEL_SIZE_INPUT=$1 # Pass L or XXL +DATASET=$2 # Pass beauty, toys, or sports + +if [[ -z "$MODEL_SIZE_INPUT" || -z "$DATASET" ]]; then + echo "Usage: sbatch embedding.sh [L|XXL] [beauty|toys|sports]" + exit 1 +fi + +MODEL_SIZE=$(echo "$MODEL_SIZE_INPUT" | tr '[:lower:]' '[:upper:]') + +if [[ "$MODEL_SIZE" == "L" || "$MODEL_SIZE" == "LARGE" ]]; then + HF_MODEL="large" +elif [[ "$MODEL_SIZE" == "XXL" ]]; then + HF_MODEL="xxl" +else + echo "Invalid model size: $MODEL_SIZE_INPUT. Choose L or XXL." + exit 1 +fi + +cd $HOME/GRID + +module purge +module load 2025 +module load Anaconda3/2025.06-1 +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +# --- FORCE ALL STORAGE & TEMP WRITES TO SCRATCH --- +export HF_HOME=/scratch-shared/$USER/hf_cache +export TMPDIR=/scratch-shared/$USER/tmp +mkdir -p $HF_HOME +mkdir -p $TMPDIR + +export CUDA_VISIBLE_DEVICES=0 +export WORLD_SIZE=1 +export OMP_NUM_THREADS=8 + +BASE=/projects/prjs2120/groups/group_08 + +python -m src.inference \ + experiment=sem_embeds_inference_flat \ + data_dir=$BASE/data/amazon_data/$DATASET \ + hydra.run.dir=/scratch-shared/$USER/embeddings/${MODEL_SIZE}/$DATASET \ + embedding_model=google/flan-t5-${HF_MODEL} \ No newline at end of file diff --git a/jobs/inference_sid.job b/jobs/inference_sid.job new file mode 100644 index 0000000..d136655 --- /dev/null +++ b/jobs/inference_sid.job @@ -0,0 +1,31 @@ +#!/bin/bash +#SBATCH --job-name=inference_sids_toys +#SBATCH --partition=gpu_a100 +#SBATCH --gpus=1 +#SBATCH --time=1:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=logs/%x-%j.out +#SBATCH --error=logs/%x-%j.err + +module purge +module load 2025 +module load Anaconda3/2025.06-1 + +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +cd $HOME/GRID + +export OMP_NUM_THREADS=8 + +BASE=/projects/prjs2120/groups/group_08 + +python -m src.inference experiment=rkmeans_inference_flat \ + data_dir=$BASE/data/amazon_data/toys \ + embedding_path=$BASE/results/embeddings/toys/pickle/merged_predictions_tensor.pt \ + embedding_dim=2048 \ + num_hierarchies=3 \ + codebook_width=256 \ + ckpt_path=$BASE/results/sid_rkmeans/toys/rkmeans_train/checkpoints/checkpoint_000_000030.ckpt \ + hydra.run.dir=$BASE/results/sid_rkmeans/toys/rkmeans_inference \ No newline at end of file diff --git a/jobs/install.job b/jobs/install.job new file mode 100644 index 0000000..00db6ff --- /dev/null +++ b/jobs/install.job @@ -0,0 +1,35 @@ +#!/bin/bash +#SBATCH --job-name=recsys_install +#SBATCH --partition=gpu_mig +#SBATCH --gpus=1 +#SBATCH --time=2:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=jobs/outputs/%x-%j.out +#SBATCH --error=jobs/outputs/%x-%j.err + +cd $HOME/FoMo + +module purge +module load 2025 +module load Anaconda3/2025.06-1 +module load CUDA/12.4.1 # Cleanly exposes nvcc to DeepSpeed + +# 2. Initialize Conda base hooks +source "$EBROOTANACONDA3/etc/profile.d/conda.sh" + +# 3. Use conda-forge to bypass Anaconda's commercial Terms of Service lock +conda create -n RecSys python=3.10 -c conda-forge -y +conda activate RecSys + +# 4. Tell DeepSpeed where your cluster CUDA module lives +export CUDA_HOME=$EBROOTCUDA + +# 5. Install PyTorch matching CUDA 12.4 +pip install torch==2.6.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124 + +# 6. Install fbgemm-gpu +pip install fbgemm-gpu==1.1.0+cu124 --index-url https://download.pytorch.org/whl/cu124 + +# 7. Install the repository dependencies file +pip install -r /home/scur1219/GRID/requirements.txt \ No newline at end of file diff --git a/jobs/logs/reproducing/L/beauty/train.log b/jobs/logs/reproducing/L/beauty/train.log new file mode 100644 index 0000000..876a9fe --- /dev/null +++ b/jobs/logs/reproducing/L/beauty/train.log @@ -0,0 +1,30 @@ +[2026-06-16 11:57:33,719][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:33,725][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:34,008][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata +[2026-06-16 11:57:34,008][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/beauty_rkmeans ++should_skip_retry=True +[2026-06-16 11:57:41,866][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:41,871][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:42,145][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-16 11:57:42,153][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-16 11:57:42,240][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-16 11:57:42,491][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-16 11:57:42,492][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,495][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,498][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,498][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,500][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,501][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-16 11:57:42,501][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.501360', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/beauty_rkmeans', '++should_skip_retry=True']} +[2026-06-16 11:57:42,501][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-16 11:57:42,502][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-16 11:57:42,503][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-16 11:57:42,505][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,507][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,508][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,554][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-16 11:57:42,950][__main__][INFO] - [rank: 0] Starting training! +[2026-06-16 11:57:44,675][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.501360', 'restarts': [], 'current_run': 0, 'used_ports': ['42809'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/beauty_rkmeans', '++should_skip_retry=True']} +[2026-06-16 17:02:05,020][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-16 17:04:12,741][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/L/beauty_rkmeans/checkpoints/checkpoint_epoch=000_step=004800.ckpt +[2026-06-16 17:04:12,767][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(7.5031), 'train/loss_step': tensor(4.6689), 'val/loss': tensor(9.5383), 'val/ndcg@5': tensor(0.0393), 'val/ndcg@10': tensor(0.0470), 'val/recall@5': tensor(0.0569), 'val/recall@10': tensor(0.0809), 'train/loss_epoch': tensor(7.5031), 'test/loss': tensor(9.8078), 'test/ndcg@5': tensor(0.0304), 'test/ndcg@10': tensor(0.0365), 'test/recall@5': tensor(0.0449), 'test/recall@10': tensor(0.0640)} +[2026-06-16 17:04:21,722][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/logs/reproducing/L/sports/train.log b/jobs/logs/reproducing/L/sports/train.log new file mode 100644 index 0000000..ec6750c --- /dev/null +++ b/jobs/logs/reproducing/L/sports/train.log @@ -0,0 +1,30 @@ +[2026-06-16 11:57:35,632][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:35,642][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:35,922][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata +[2026-06-16 11:57:35,922][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/sports_rkmeans ++should_skip_retry=True +[2026-06-16 11:57:44,389][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:44,394][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:44,670][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-16 11:57:44,679][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-16 11:57:44,801][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-16 11:57:45,077][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-16 11:57:45,077][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,081][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,083][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,084][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,086][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,086][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-16 11:57:45,086][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:45.086844', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-16 11:57:45,087][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-16 11:57:45,087][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-16 11:57:45,089][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-16 11:57:45,091][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,093][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,093][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,141][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-16 11:57:45,571][__main__][INFO] - [rank: 0] Starting training! +[2026-06-16 11:57:47,411][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:45.086844', 'restarts': [], 'current_run': 0, 'used_ports': ['47843'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-16 17:28:49,970][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-16 17:32:11,383][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/L/sports_rkmeans/checkpoints/checkpoint_epoch=000_step=004100.ckpt +[2026-06-16 17:32:11,391][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(9.3024), 'train/loss_step': tensor(8.0710), 'val/loss': tensor(9.2528), 'val/ndcg@5': tensor(0.0193), 'val/ndcg@10': tensor(0.0241), 'val/recall@5': tensor(0.0294), 'val/recall@10': tensor(0.0440), 'train/loss_epoch': tensor(9.3024), 'test/loss': tensor(9.5419), 'test/ndcg@5': tensor(0.0147), 'test/ndcg@10': tensor(0.0182), 'test/recall@5': tensor(0.0223), 'test/recall@10': tensor(0.0331)} +[2026-06-16 17:32:15,236][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/logs/reproducing/L/toys/train.log b/jobs/logs/reproducing/L/toys/train.log new file mode 100644 index 0000000..124493a --- /dev/null +++ b/jobs/logs/reproducing/L/toys/train.log @@ -0,0 +1,30 @@ +[2026-06-16 11:57:50,782][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:50,792][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:51,076][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata +[2026-06-16 11:57:51,077][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/toys_rkmeans ++should_skip_retry=True +[2026-06-16 11:57:59,361][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:59,367][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:59,639][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-16 11:57:59,656][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-16 11:57:59,809][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-16 11:58:00,096][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-16 11:58:00,096][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:58:00,100][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:58:00,102][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:58:00,103][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:58:00,105][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:58:00,106][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-16 11:58:00,106][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:58:00.106169', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/toys_rkmeans', '++should_skip_retry=True']} +[2026-06-16 11:58:00,107][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-16 11:58:00,107][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-16 11:58:00,109][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-16 11:58:00,111][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:58:00,113][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:58:00,113][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:58:00,171][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-16 11:58:01,130][__main__][INFO] - [rank: 0] Starting training! +[2026-06-16 11:58:02,558][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/toys_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:58:00.106169', 'restarts': [], 'current_run': 0, 'used_ports': ['36927'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/toys_rkmeans', '++should_skip_retry=True']} +[2026-06-16 15:27:48,918][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-16 15:29:38,187][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/L/toys_rkmeans/checkpoints/checkpoint_epoch=000_step=003300.ckpt +[2026-06-16 15:29:38,235][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(7.8262), 'train/loss_step': tensor(6.8433), 'val/loss': tensor(9.8689), 'val/ndcg@5': tensor(0.0340), 'val/ndcg@10': tensor(0.0418), 'val/recall@5': tensor(0.0499), 'val/recall@10': tensor(0.0740), 'train/loss_epoch': tensor(7.8262), 'test/loss': tensor(9.9868), 'test/ndcg@5': tensor(0.0256), 'test/ndcg@10': tensor(0.0322), 'test/recall@5': tensor(0.0400), 'test/recall@10': tensor(0.0607)} +[2026-06-16 15:29:48,727][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/logs/reproducing/XXL/beauty/train.log b/jobs/logs/reproducing/XXL/beauty/train.log new file mode 100644 index 0000000..b347359 --- /dev/null +++ b/jobs/logs/reproducing/XXL/beauty/train.log @@ -0,0 +1,30 @@ +[2026-06-16 11:57:35,636][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:35,642][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:35,921][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata +[2026-06-16 11:57:35,921][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans ++should_skip_retry=True +[2026-06-16 11:57:44,410][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:44,415][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:44,690][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-16 11:57:44,693][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-16 11:57:44,801][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-16 11:57:45,076][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-16 11:57:45,076][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,080][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,082][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,083][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:45,085][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,085][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-16 11:57:45,085][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:45.085780', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans', '++should_skip_retry=True']} +[2026-06-16 11:57:45,086][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-16 11:57:45,086][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-16 11:57:45,088][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-16 11:57:45,090][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,092][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,092][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:45,141][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-16 11:57:45,571][__main__][INFO] - [rank: 0] Starting training! +[2026-06-16 11:57:47,361][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:45.085780', 'restarts': [], 'current_run': 0, 'used_ports': ['58781'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/beauty_rkmeans', '++should_skip_retry=True']} +[2026-06-16 15:51:22,627][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-16 15:53:25,977][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/XXL/beauty_rkmeans/checkpoints/checkpoint_epoch=000_step=003500.ckpt +[2026-06-16 15:53:26,005][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(7.6635), 'train/loss_step': tensor(7.8335), 'val/loss': tensor(9.0442), 'val/ndcg@5': tensor(0.0382), 'val/ndcg@10': tensor(0.0467), 'val/recall@5': tensor(0.0575), 'val/recall@10': tensor(0.0838), 'train/loss_epoch': tensor(7.6635), 'test/loss': tensor(9.2698), 'test/ndcg@5': tensor(0.0262), 'test/ndcg@10': tensor(0.0334), 'test/recall@5': tensor(0.0404), 'test/recall@10': tensor(0.0628)} +[2026-06-16 15:53:34,482][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/logs/reproducing/XXL/sports/train.log b/jobs/logs/reproducing/XXL/sports/train.log new file mode 100644 index 0000000..d9e762a --- /dev/null +++ b/jobs/logs/reproducing/XXL/sports/train.log @@ -0,0 +1,30 @@ +[2026-06-16 11:57:34,214][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:34,220][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:34,496][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata +[2026-06-16 11:57:34,497][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans ++should_skip_retry=True +[2026-06-16 11:57:42,317][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:42,322][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:42,596][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-16 11:57:42,606][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-16 11:57:42,704][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-16 11:57:42,951][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-16 11:57:42,951][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,954][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,957][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,957][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,960][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,960][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-16 11:57:42,960][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.960440', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-16 11:57:42,960][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-16 11:57:42,961][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-16 11:57:42,963][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-16 11:57:42,964][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,966][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,967][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:43,013][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-16 11:57:43,419][__main__][INFO] - [rank: 0] Starting training! +[2026-06-16 11:57:45,247][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.960440', 'restarts': [], 'current_run': 0, 'used_ports': ['33033'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-16 17:26:28,230][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-16 17:29:47,164][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/XXL/sports_rkmeans/checkpoints/checkpoint_epoch=000_step=004100.ckpt +[2026-06-16 17:29:47,174][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(9.1292), 'train/loss_step': tensor(7.8743), 'val/loss': tensor(9.1621), 'val/ndcg@5': tensor(0.0202), 'val/ndcg@10': tensor(0.0250), 'val/recall@5': tensor(0.0298), 'val/recall@10': tensor(0.0447), 'train/loss_epoch': tensor(9.1292), 'test/loss': tensor(9.4324), 'test/ndcg@5': tensor(0.0153), 'test/ndcg@10': tensor(0.0197), 'test/recall@5': tensor(0.0230), 'test/recall@10': tensor(0.0367)} +[2026-06-16 17:29:53,575][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/logs/reproducing/XXL/toys/train.log b/jobs/logs/reproducing/XXL/toys/train.log new file mode 100644 index 0000000..5133d91 --- /dev/null +++ b/jobs/logs/reproducing/XXL/toys/train.log @@ -0,0 +1,30 @@ +[2026-06-16 11:57:34,222][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:34,228][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:34,503][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata +[2026-06-16 11:57:34,503][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans ++should_skip_retry=True +[2026-06-16 11:57:42,319][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-16 11:57:42,324][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-16 11:57:42,597][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-16 11:57:42,606][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-16 11:57:42,708][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-16 11:57:42,952][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-16 11:57:42,952][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,955][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,958][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,958][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-16 11:57:42,961][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,961][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-16 11:57:42,961][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.961393', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans', '++should_skip_retry=True']} +[2026-06-16 11:57:42,961][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-16 11:57:42,962][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-16 11:57:42,963][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-16 11:57:42,965][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,967][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:42,967][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json +[2026-06-16 11:57:43,013][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-16 11:57:43,419][__main__][INFO] - [rank: 0] Starting training! +[2026-06-16 11:57:45,258][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.961393', 'restarts': [], 'current_run': 0, 'used_ports': ['33453'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/XXL/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/XXL/toys_rkmeans', '++should_skip_retry=True']} +[2026-06-16 16:17:12,668][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-16 16:19:01,920][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/XXL/toys_rkmeans/checkpoints/checkpoint_epoch=000_step=004300.ckpt +[2026-06-16 16:19:01,945][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(7.3486), 'train/loss_step': tensor(6.5392), 'val/loss': tensor(9.9951), 'val/ndcg@5': tensor(0.0367), 'val/ndcg@10': tensor(0.0440), 'val/recall@5': tensor(0.0536), 'val/recall@10': tensor(0.0764), 'train/loss_epoch': tensor(7.3486), 'test/loss': tensor(10.2606), 'test/ndcg@5': tensor(0.0264), 'test/ndcg@10': tensor(0.0333), 'test/recall@5': tensor(0.0409), 'test/recall@10': tensor(0.0624)} +[2026-06-16 16:19:09,630][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/logs/reproducing/beauty/train.log b/jobs/logs/reproducing/beauty/train.log new file mode 100644 index 0000000..470b42e --- /dev/null +++ b/jobs/logs/reproducing/beauty/train.log @@ -0,0 +1,30 @@ +[2026-06-10 10:08:01,933][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-10 10:08:01,940][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-10 10:08:02,223][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata +[2026-06-10 10:08:02,223][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/beauty ++should_skip_retry=True +[2026-06-10 10:08:10,587][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-10 10:08:10,593][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-10 10:08:10,868][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-10 10:08:10,877][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-10 10:08:10,988][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-10 10:08:11,271][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-10 10:08:11,271][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 10:08:11,275][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 10:08:11,277][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 10:08:11,278][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 10:08:11,280][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json +[2026-06-10 10:08:11,281][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-10 10:08:11,281][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json. {'start_time': '2026-06-10T10:08:11.281334', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/beauty', '++should_skip_retry=True']} +[2026-06-10 10:08:11,323][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-10 10:08:11,323][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-10 10:08:11,325][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-10 10:08:11,328][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json +[2026-06-10 10:08:11,331][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json +[2026-06-10 10:08:11,331][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json +[2026-06-10 10:08:11,381][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-10 10:08:11,803][__main__][INFO] - [rank: 0] Starting training! +[2026-06-10 10:08:14,827][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/beauty/metadata/restart_metadata.json. {'start_time': '2026-06-10T10:08:11.281334', 'restarts': [], 'current_run': 0, 'used_ports': ['44417'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/beauty', '++should_skip_retry=True']} +[2026-06-10 15:04:16,480][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-10 15:06:30,519][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/work5/0/prjs2120/groups/group_08/results/tiger/beauty/checkpoints/checkpoint_epoch=000_step=004500.ckpt +[2026-06-10 15:06:30,642][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(7.4643), 'train/loss_step': tensor(7.2329), 'val/loss': tensor(9.4356), 'val/ndcg@5': tensor(0.0389), 'val/ndcg@10': tensor(0.0466), 'val/recall@5': tensor(0.0571), 'val/recall@10': tensor(0.0809), 'train/loss_epoch': tensor(7.4643), 'test/loss': tensor(9.6659), 'test/ndcg@5': tensor(0.0304), 'test/ndcg@10': tensor(0.0369), 'test/recall@5': tensor(0.0452), 'test/recall@10': tensor(0.0655)} +[2026-06-10 15:06:39,095][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/logs/reproducing/sports/train.log b/jobs/logs/reproducing/sports/train.log new file mode 100644 index 0000000..7638350 --- /dev/null +++ b/jobs/logs/reproducing/sports/train.log @@ -0,0 +1,97 @@ +[2026-06-10 17:54:07,168][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-10 17:54:07,175][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-10 17:54:07,457][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata +[2026-06-10 17:54:07,457][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans ++should_skip_retry=True +[2026-06-10 17:54:15,436][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-10 17:54:15,443][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-10 17:54:15,715][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-10 17:54:15,725][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-10 17:54:16,001][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-10 17:54:16,296][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-10 17:54:16,297][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:16,300][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:16,302][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:16,303][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:16,305][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:16,306][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-10 17:54:16,306][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:16.306267', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-10 17:54:16,376][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-10 17:54:16,376][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-10 17:54:16,378][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-10 17:54:16,381][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:16,383][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:16,383][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:16,433][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-10 17:54:17,069][__main__][INFO] - [rank: 0] Starting training! +[2026-06-10 17:54:19,682][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:16.306267', 'restarts': [], 'current_run': 0, 'used_ports': ['60229'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-10 23:13:50,592][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-10 23:17:05,964][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/work5/0/prjs2120/groups/group_08/results/tiger/sports_rkmeans/checkpoints/checkpoint_epoch=000_step=004000.ckpt +[2026-06-10 23:17:05,993][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(9.2376), 'train/loss_step': tensor(7.9633), 'val/loss': tensor(9.2298), 'val/ndcg@5': tensor(0.0195), 'val/ndcg@10': tensor(0.0242), 'val/recall@5': tensor(0.0296), 'val/recall@10': tensor(0.0443), 'train/loss_epoch': tensor(9.2376), 'test/loss': tensor(9.5222), 'test/ndcg@5': tensor(0.0153), 'test/ndcg@10': tensor(0.0190), 'test/recall@5': tensor(0.0231), 'test/recall@10': tensor(0.0346)} +[2026-06-10 23:17:11,438][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. +[2026-06-11 11:03:32,516][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-11 11:03:32,544][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-11 11:03:32,823][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata +[2026-06-11 11:03:32,824][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1256/.conda/envs/grid/bin/python /gpfs/home4/scur1256/GRID/src/train.py experiment=tiger_train_flat_dec_only data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans ++should_skip_retry=True +[2026-06-11 11:03:40,640][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-11 11:03:40,646][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-11 11:03:40,918][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-11 11:03:40,928][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-11 11:03:41,226][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-11 11:03:47,839][src.utils.restart_job][ERROR] - [rank: 0] Job failed with return code 1 +[2026-06-11 11:03:47,839][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:03:47,884][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:03:47,884][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:03:47,884][src.utils.restart_job][ERROR] - [rank: 0] Metadata run count 0 does not match current run count 1.This can happen if the exception does not trigger the callback on_exception. Cleaning up andupdating metadata before retrying. +[2026-06-11 11:03:47,900][src.utils.restart_job_utils][INFO] - [rank: 0] Clearing CUDA cache +[2026-06-11 11:03:47,900][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:03:47,901][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:03:47,901][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:16.306267', 'restarts': [], 'current_run': 1, 'used_ports': ['60229'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-11 11:03:47,901][src.utils.restart_job][INFO] - [rank: 0] Retrying in 5 seconds. Attempt 2/4 +[2026-06-11 11:03:52,907][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 2/4): /home/scur1256/.conda/envs/grid/bin/python /gpfs/home4/scur1256/GRID/src/train.py experiment=tiger_train_flat_dec_only data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans ++should_skip_retry=True +[2026-06-11 11:04:00,632][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-11 11:04:00,637][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-11 11:04:00,910][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-11 11:04:00,913][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-11 11:04:00,973][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-11 11:04:02,918][src.utils.restart_job][ERROR] - [rank: 0] Job failed with return code 1 +[2026-06-11 11:04:02,918][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:02,918][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:02,919][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 1 from metadata /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:02,919][src.utils.restart_job][ERROR] - [rank: 0] Metadata run count 1 does not match current run count 2.This can happen if the exception does not trigger the callback on_exception. Cleaning up andupdating metadata before retrying. +[2026-06-11 11:04:02,919][src.utils.restart_job_utils][INFO] - [rank: 0] Clearing CUDA cache +[2026-06-11 11:04:02,919][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:02,920][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:02,920][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:16.306267', 'restarts': [], 'current_run': 2, 'used_ports': ['60229'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-11 11:04:02,920][src.utils.restart_job][INFO] - [rank: 0] Retrying in 5 seconds. Attempt 3/4 +[2026-06-11 11:04:07,925][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 3/4): /home/scur1256/.conda/envs/grid/bin/python /gpfs/home4/scur1256/GRID/src/train.py experiment=tiger_train_flat_dec_only data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans ++should_skip_retry=True +[2026-06-11 11:04:15,545][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-11 11:04:15,550][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-11 11:04:15,824][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-11 11:04:15,830][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-11 11:04:15,889][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-11 11:04:17,936][src.utils.restart_job][ERROR] - [rank: 0] Job failed with return code 1 +[2026-06-11 11:04:17,937][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:17,937][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:17,937][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 2 from metadata /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:17,937][src.utils.restart_job][ERROR] - [rank: 0] Metadata run count 2 does not match current run count 3.This can happen if the exception does not trigger the callback on_exception. Cleaning up andupdating metadata before retrying. +[2026-06-11 11:04:17,938][src.utils.restart_job_utils][INFO] - [rank: 0] Clearing CUDA cache +[2026-06-11 11:04:17,938][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:17,938][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:17,938][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:16.306267', 'restarts': [], 'current_run': 3, 'used_ports': ['60229'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-11 11:04:17,941][src.utils.restart_job][INFO] - [rank: 0] Retrying in 5 seconds. Attempt 4/4 +[2026-06-11 11:04:22,947][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 4/4): /home/scur1256/.conda/envs/grid/bin/python /gpfs/home4/scur1256/GRID/src/train.py experiment=tiger_train_flat_dec_only data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans ++should_skip_retry=True +[2026-06-11 11:04:30,730][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-11 11:04:30,735][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-11 11:04:31,009][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-11 11:04:31,012][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-11 11:04:31,073][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-11 11:04:32,958][src.utils.restart_job][ERROR] - [rank: 0] Job failed with return code 1 +[2026-06-11 11:04:32,958][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:32,958][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:32,959][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 3 from metadata /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:32,959][src.utils.restart_job][ERROR] - [rank: 0] Metadata run count 3 does not match current run count 4.This can happen if the exception does not trigger the callback on_exception. Cleaning up andupdating metadata before retrying. +[2026-06-11 11:04:32,959][src.utils.restart_job_utils][INFO] - [rank: 0] Clearing CUDA cache +[2026-06-11 11:04:32,959][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:32,959][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json +[2026-06-11 11:04:32,960][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:16.306267', 'restarts': [], 'current_run': 4, 'used_ports': ['60229'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/sports_rkmeans', '++should_skip_retry=True']} +[2026-06-11 11:04:32,960][src.utils.restart_job][ERROR] - [rank: 0] Job failed after 3 retries. diff --git a/jobs/logs/reproducing/toys/train.log b/jobs/logs/reproducing/toys/train.log new file mode 100644 index 0000000..8a36678 --- /dev/null +++ b/jobs/logs/reproducing/toys/train.log @@ -0,0 +1,30 @@ +[2026-06-10 17:54:19,912][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-10 17:54:19,921][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-10 17:54:20,206][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata +[2026-06-10 17:54:20,206][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans ++should_skip_retry=True +[2026-06-10 17:54:28,398][src.utils.utils][INFO] - [rank: 0] Enforcing tags! +[2026-06-10 17:54:28,406][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! +[2026-06-10 17:54:28,681][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic. +[2026-06-10 17:54:28,710][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule +[2026-06-10 17:54:28,947][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model +[2026-06-10 17:54:29,250][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks... +[2026-06-10 17:54:29,250][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:29,255][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:29,257][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:29,258][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback +[2026-06-10 17:54:29,261][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:29,261][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json. Creating empty metadata. +[2026-06-10 17:54:29,262][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:29.262053', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans', '++should_skip_retry=True']} +[2026-06-10 17:54:29,263][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers... +[2026-06-10 17:54:29,264][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger +[2026-06-10 17:54:29,266][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer +[2026-06-10 17:54:29,268][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:29,271][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:29,271][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json +[2026-06-10 17:54:29,328][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters! +[2026-06-10 17:54:30,476][__main__][INFO] - [rank: 0] Starting training! +[2026-06-10 17:54:32,696][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-10T17:54:29.262053', 'restarts': [], 'current_run': 0, 'used_ports': ['48187'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/toys', 'semantic_id_path=/projects/prjs2120/groups/group_08/results/sid_rkmeans/toys/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/projects/prjs2120/groups/group_08/results/tiger/toys_rkmeans', '++should_skip_retry=True']} +[2026-06-10 21:00:04,420][__main__][INFO] - [rank: 0] Starting testing! +[2026-06-10 21:01:53,743][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/work5/0/prjs2120/groups/group_08/results/tiger/toys_rkmeans/checkpoints/checkpoint_epoch=000_step=002800.ckpt +[2026-06-10 21:01:53,791][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(7.9583), 'train/loss_step': tensor(6.7817), 'val/loss': tensor(9.4167), 'val/ndcg@5': tensor(0.0345), 'val/ndcg@10': tensor(0.0423), 'val/recall@5': tensor(0.0525), 'val/recall@10': tensor(0.0769), 'train/loss_epoch': tensor(7.9583), 'test/loss': tensor(9.6428), 'test/ndcg@5': tensor(0.0265), 'test/ndcg@10': tensor(0.0329), 'test/recall@5': tensor(0.0410), 'test/recall@10': tensor(0.0605)} +[2026-06-10 21:02:06,365][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully. diff --git a/jobs/rkmeans_inference.sh b/jobs/rkmeans_inference.sh new file mode 100644 index 0000000..c2452ef --- /dev/null +++ b/jobs/rkmeans_inference.sh @@ -0,0 +1,51 @@ +#!/bin/bash +#SBATCH --job-name=infer_sids +#SBATCH --partition=gpu_a100 +#SBATCH --gpus=1 +#SBATCH --time=1:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=logs/%x-%j.out +#SBATCH --error=logs/%x-%j.err + +MODEL_SIZE=$1 +DATASET=$2 +CKPT_NAME=$3 + +if [[ -z "$MODEL_SIZE" || -z "$DATASET" || -z "$CKPT_NAME" ]]; then + echo "Usage: sbatch rkmeans_inference.sh [L|XXL] [beauty|toys|sports] [checkpoint_name.ckpt]" + exit 1 +fi + +if [ "$MODEL_SIZE" = "L" ]; then + DIM=1024 +elif [ "$MODEL_SIZE" = "XXL" ]; then + DIM=4096 +else + echo "Invalid model size. Choose L or XXL." + exit 1 +fi + +module purge +module load 2025 +module load Anaconda3/2025.06-1 +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +cd $HOME/GRID +export OMP_NUM_THREADS=8 +BASE_PROJECT=/projects/prjs2120/groups/group_08 +BASE_SCRATCH=/scratch-shared/$USER + +# Keep metadata/logging directory in project folder (very lightweight text files) +mkdir -p $BASE_PROJECT/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_inference/metadata + +# Run inference utilizing completely isolated scratch paths +python -m src.inference experiment=rkmeans_inference_flat \ + data_dir=$BASE_PROJECT/data/amazon_data/$DATASET \ + embedding_path=$BASE_SCRATCH/embeddings/${MODEL_SIZE}/${DATASET}/pickle/merged_predictions_tensor.pt \ + embedding_dim=$DIM \ + num_hierarchies=3 \ + codebook_width=256 \ + ckpt_path=$BASE_SCRATCH/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_train/checkpoints/$CKPT_NAME \ + hydra.run.dir=$BASE_SCRATCH/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_inference \ No newline at end of file diff --git a/jobs/rkmeans_train.sh b/jobs/rkmeans_train.sh new file mode 100644 index 0000000..0ef2acc --- /dev/null +++ b/jobs/rkmeans_train.sh @@ -0,0 +1,52 @@ +#!/bin/bash +#SBATCH --job-name=train_sids +#SBATCH --partition=gpu_h100 +#SBATCH --gpus=1 +#SBATCH --time=1:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=logs/%x-%j.out +#SBATCH --error=logs/%x-%j.err + +MODEL_SIZE=$1 +DATASET=$2 + +if [[ -z "$MODEL_SIZE" || -z "$DATASET" ]]; then + echo "Usage: sbatch rkmeans_train.sh [L|XXL] [beauty|toys|sports]" + exit 1 +fi + +if [ "$MODEL_SIZE" = "L" ]; then + DIM=1024 +elif [ "$MODEL_SIZE" = "XXL" ]; then + DIM=4096 +else + echo "Invalid model size. Choose L or XXL." + exit 1 +fi + +module purge +module load 2025 +module load Anaconda3/2025.06-1 +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +cd $HOME/GRID +export OMP_NUM_THREADS=8 +BASE_PROJECT=/projects/prjs2120/groups/group_08 +BASE_SCRATCH=/scratch-shared/$USER + +# Point directly to your scratch embeddings +EMBED_PATH=$BASE_SCRATCH/embeddings/${MODEL_SIZE}/${DATASET}/pickle/merged_predictions_tensor.pt + +# Ensure the log/metadata structure exists in the project folder (lightweight text files) +mkdir -p $BASE_PROJECT/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_train/metadata + +# Force the heavy checkpoint outputs into your scratch playground +python -m src.train experiment=rkmeans_train_flat \ + data_dir=$BASE_PROJECT/data/amazon_data/$DATASET \ + embedding_path=$EMBED_PATH \ + embedding_dim=$DIM \ + num_hierarchies=3 \ + codebook_width=256 \ + hydra.run.dir=$BASE_SCRATCH/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_train \ No newline at end of file diff --git a/jobs/tiger_inference.job b/jobs/tiger_inference.job new file mode 100644 index 0000000..2f16819 --- /dev/null +++ b/jobs/tiger_inference.job @@ -0,0 +1,33 @@ +#!/bin/bash +#SBATCH --job-name=tiger_inf_beauty +#SBATCH --partition=gpu_a100 +#SBATCH --gpus=1 +#SBATCH --time=2:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=logs/tiger_inf_%j.out +#SBATCH --error=logs/tiger_inf_%j.err + +module purge +module load 2025 +module load Anaconda3/2025.06-1 + +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +export OMP_NUM_THREADS=8 + +cd $HOME/GRID + +BASE=/projects/prjs2120/groups/group_08 + + +python -m src.inference experiment=tiger_inference_flat \ + data_dir=$BASE/data/amazon_data/beauty \ + semantic_id_path=$BASE/results/sid/beauty_v2/rkmeans_inference/pickle/merged_predictions_tensor.pt \ + ckpt_path=$BASE/results/tiger/beauty_rkmeans/checkpoints/checkpoint_epoch=000_step=004500.ckpt \ # this can be found in the log dir for training GR models + num_hierarchies=4 \ + hydra.run.dir=$BASE/results/tiger/beauty_rkmeans/inference \ + + + diff --git a/jobs/tiger_train.sh b/jobs/tiger_train.sh new file mode 100644 index 0000000..2659fbc --- /dev/null +++ b/jobs/tiger_train.sh @@ -0,0 +1,42 @@ +#!/bin/bash +#SBATCH --job-name=tiger_train +#SBATCH --partition=gpu_a100 +#SBATCH --gpus=1 +#SBATCH --time=6:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=logs/tiger_train_%j.out +#SBATCH --error=logs/tiger_train_%j.err + +MODEL_SIZE_INPUT=$1 # Pass L or XXL +DATASET=$2 # Pass beauty, toys, or sports + +if [[ -z "$MODEL_SIZE_INPUT" || -z "$DATASET" ]]; then + echo "Usage: sbatch tiger_train.sh [L|XXL] [beauty|toys|sports]" + exit 1 +fi + +# Standardise case handling just like your other scripts +MODEL_SIZE=$(echo "$MODEL_SIZE_INPUT" | tr '[:lower:]' '[:upper:]') + +module purge +module load 2025 +module load Anaconda3/2025.06-1 +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +export OMP_NUM_THREADS=8 +cd $HOME/GRID + +BASE_PROJECT=/projects/prjs2120/groups/group_08 +BASE_SCRATCH=/scratch-shared/$USER + +# Point semantic_id_path to where rkmeans_inference saved the outputs on scratch +SEMANTIC_ID_PATH=$BASE_SCRATCH/results/sid_rkmeans/${MODEL_SIZE}/${DATASET}/rkmeans_inference/pickle/merged_predictions_tensor.pt + +# Run the final generative recommendation training completely on scratch +python -m src.train experiment=tiger_train_flat \ + data_dir=$BASE_PROJECT/data/amazon_data/$DATASET \ + semantic_id_path=$SEMANTIC_ID_PATH \ + num_hierarchies=4 \ + hydra.run.dir=$BASE_SCRATCH/results/tiger/${MODEL_SIZE}/${DATASET}_rkmeans \ No newline at end of file diff --git a/jobs/train_sid.job b/jobs/train_sid.job new file mode 100644 index 0000000..f28efdf --- /dev/null +++ b/jobs/train_sid.job @@ -0,0 +1,31 @@ +#!/bin/bash +#SBATCH --job-name=train_sids_beauty +#SBATCH --partition=gpu_a100 +#SBATCH --gpus=1 +#SBATCH --time=1:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=logs/%x-%j.out +#SBATCH --error=logs/%x-%j.err + +module purge +module load 2025 +module load Anaconda3/2025.06-1 + +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +cd $HOME/GRID + +export OMP_NUM_THREADS=8 + +BASE=/projects/prjs2120/groups/group_08 + +python -m src.train experiment=rkmeans_train_flat \ + data_dir=$BASE/data/amazon_data/beauty \ + embedding_path=$BASE/results/embeddings/beauty_v2/pickle/merged_predictions_tensor.pt \ + embedding_dim=2048 \ + num_hierarchies=3 \ + codebook_width=256 \ + hydra.run.dir=$BASE/results/sid_rkmeans/beauty/rkmeans_train + diff --git a/jobs/train_tiger.job b/jobs/train_tiger.job new file mode 100644 index 0000000..3ea7f5f --- /dev/null +++ b/jobs/train_tiger.job @@ -0,0 +1,29 @@ +#!/bin/bash +#SBATCH --job-name=tiger_train_sports +#SBATCH --partition=gpu_h100 +#SBATCH --gpus=1 +#SBATCH --time=10:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G +#SBATCH --output=logs/tiger_train_%j.out +#SBATCH --error=logs/tiger_train_%j.err + +module purge +module load 2025 +module load Anaconda3/2025.06-1 + +source $(conda info --base)/etc/profile.d/conda.sh +conda activate RecSys + +export OMP_NUM_THREADS=8 + +cd $HOME/GRID + +BASE=/projects/prjs2120/groups/group_08 +BASE_SCRATCH=/scratch-shared/$USER + +python -m src.train experiment=tiger_train_flat \ + data_dir=$BASE/data/amazon_data/sports \ + semantic_id_path=$BASE/results/sid_rkmeans/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt \ + num_hierarchies=4 \ + hydra.run.dir=$BASE_SCRATCH/results/tiger_default_2/sports_rkmeans \ No newline at end of file diff --git a/src/utils/inference_utils.py b/src/utils/inference_utils.py index cbde1c3..5d43876 100755 --- a/src/utils/inference_utils.py +++ b/src/utils/inference_utils.py @@ -233,14 +233,20 @@ def on_predict_end( if self.should_merge_files_on_main: # if we use multiple workers, we need to wait for all of them to finish writing # before merging the files - if trainer.global_rank != None: + if ( + torch.distributed.is_available() + and torch.distributed.is_initialized() + ): torch.distributed.barrier() if self.global_rank == 0: log.info("Merging pickle files on main process.") self._merge_files() # other processes can continue after merging - if trainer.global_rank != None: + if ( + torch.distributed.is_available() + and torch.distributed.is_initialized() + ): torch.distributed.barrier() # conducting post-processing functions on the files @@ -253,7 +259,10 @@ def on_predict_end( process_func["function"](file_path) else: process_func["function"](file_path) - if trainer.global_rank != None: + if ( + torch.distributed.is_available() + and torch.distributed.is_initialized() + ): torch.distributed.barrier() def _merge_files(self):