Reproducing

.gitignore

@@ -56,7 +56,7 @@ cover/
 *.pot
 
 # Django stuff:
-*.log
+# *.log
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
@@ -191,4 +191,9 @@ cython_debug/
 #  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
 #  refer to https://docs.cursor.com/context/ignore-files
 .cursorignore
-.cursorindexingignore
+.cursorindexingignore
+
+# job files
+/jobs/outputs
+*.err
+*.out

jobs/create_embeddings.job

@@ -0,0 +1,28 @@
+#!/bin/bash
+#SBATCH --job-name=embeddings
+#SBATCH --partition=gpu_h100
+#SBATCH --gpus=1
+#SBATCH --time=2:00:00
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=32G
+#SBATCH --output=jobs/outputs/%x-%j.out
+#SBATCH --error=jobs/outputs/%x-%j.err
+
+cd $HOME/GRID
+
+module purge
+module load 2025
+module load Anaconda3/2025.06-1
+
+source $(conda info --base)/etc/profile.d/conda.sh
+conda activate RecSys
+
+export CUDA_VISIBLE_DEVICES=0
+export WORLD_SIZE=1
+export OMP_NUM_THREADS=8
+
+python -m src.inference \
+    experiment=sem_embeds_inference_flat \
+    data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports \
+    hydra.run.dir=/projects/prjs2120/groups/group_08/results/embeddings/sports
+

jobs/embedding.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+#SBATCH --job-name=embeds
+#SBATCH --partition=gpu_h100
+#SBATCH --gpus=1
+#SBATCH --time=2:00:00
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=32G
+#SBATCH --output=jobs/outputs/%x-%j.out
+#SBATCH --error=jobs/outputs/%x-%j.err
+
+MODEL_SIZE_INPUT=$1  # Pass L or XXL
+DATASET=$2           # Pass beauty, toys, or sports
+
+if [[ -z "$MODEL_SIZE_INPUT" || -z "$DATASET" ]]; then
+    echo "Usage: sbatch embedding.sh [L|XXL] [beauty|toys|sports]"
+    exit 1
+fi
+
+MODEL_SIZE=$(echo "$MODEL_SIZE_INPUT" | tr '[:lower:]' '[:upper:]')
+
+if [[ "$MODEL_SIZE" == "L" || "$MODEL_SIZE" == "LARGE" ]]; then
+    HF_MODEL="large"
+elif [[ "$MODEL_SIZE" == "XXL" ]]; then
+    HF_MODEL="xxl"
+else
+    echo "Invalid model size: $MODEL_SIZE_INPUT. Choose L or XXL."
+    exit 1
+fi
+
+cd $HOME/GRID
+
+module purge
+module load 2025
+module load Anaconda3/2025.06-1
+source $(conda info --base)/etc/profile.d/conda.sh
+conda activate RecSys
+
+# --- FORCE ALL STORAGE & TEMP WRITES TO SCRATCH ---
+export HF_HOME=/scratch-shared/$USER/hf_cache
+export TMPDIR=/scratch-shared/$USER/tmp
+mkdir -p $HF_HOME
+mkdir -p $TMPDIR
+
+export CUDA_VISIBLE_DEVICES=0
+export WORLD_SIZE=1
+export OMP_NUM_THREADS=8
+
+BASE=/projects/prjs2120/groups/group_08
+
+python -m src.inference \
+    experiment=sem_embeds_inference_flat \
+    data_dir=$BASE/data/amazon_data/$DATASET \
+    hydra.run.dir=/scratch-shared/$USER/embeddings/${MODEL_SIZE}/$DATASET \
+    embedding_model=google/flan-t5-${HF_MODEL}

jobs/inference_sid.job

@@ -0,0 +1,31 @@
+#!/bin/bash
+#SBATCH --job-name=inference_sids_toys
+#SBATCH --partition=gpu_a100
+#SBATCH --gpus=1
+#SBATCH --time=1:00:00
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=32G
+#SBATCH --output=logs/%x-%j.out
+#SBATCH --error=logs/%x-%j.err
+
+module purge
+module load 2025
+module load Anaconda3/2025.06-1
+
+source $(conda info --base)/etc/profile.d/conda.sh
+conda activate RecSys
+
+cd $HOME/GRID
+
+export OMP_NUM_THREADS=8
+
+BASE=/projects/prjs2120/groups/group_08
+
+python -m src.inference experiment=rkmeans_inference_flat \
+    data_dir=$BASE/data/amazon_data/toys \
+    embedding_path=$BASE/results/embeddings/toys/pickle/merged_predictions_tensor.pt \
+    embedding_dim=2048 \
+    num_hierarchies=3 \
+    codebook_width=256 \
+    ckpt_path=$BASE/results/sid_rkmeans/toys/rkmeans_train/checkpoints/checkpoint_000_000030.ckpt \
+    hydra.run.dir=$BASE/results/sid_rkmeans/toys/rkmeans_inference

jobs/install.job

@@ -0,0 +1,35 @@
+#!/bin/bash
+#SBATCH --job-name=recsys_install
+#SBATCH --partition=gpu_mig
+#SBATCH --gpus=1
+#SBATCH --time=2:00:00
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=32G
+#SBATCH --output=jobs/outputs/%x-%j.out
+#SBATCH --error=jobs/outputs/%x-%j.err
+
+cd $HOME/FoMo
+
+module purge
+module load 2025
+module load Anaconda3/2025.06-1
+module load CUDA/12.4.1                  # Cleanly exposes nvcc to DeepSpeed
+
+# 2. Initialize Conda base hooks
+source "$EBROOTANACONDA3/etc/profile.d/conda.sh"
+
+# 3. Use conda-forge to bypass Anaconda's commercial Terms of Service lock
+conda create -n RecSys python=3.10 -c conda-forge -y
+conda activate RecSys
+
+# 4. Tell DeepSpeed where your cluster CUDA module lives
+export CUDA_HOME=$EBROOTCUDA
+
+# 5. Install PyTorch matching CUDA 12.4
+pip install torch==2.6.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124
+
+# 6. Install fbgemm-gpu
+pip install fbgemm-gpu==1.1.0+cu124 --index-url https://download.pytorch.org/whl/cu124
+
+# 7. Install the repository dependencies file
+pip install -r /home/scur1219/GRID/requirements.txt

jobs/logs/reproducing/L/beauty/train.log

@@ -0,0 +1,30 @@
+[2026-06-16 11:57:33,719][src.utils.utils][INFO] - [rank: 0] Enforcing tags! <cfg.extras.enforce_tags=True>
+[2026-06-16 11:57:33,725][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! <cfg.extras.print_config=True>
+[2026-06-16 11:57:34,008][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata
+[2026-06-16 11:57:34,008][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/beauty_rkmeans ++should_skip_retry=True
+[2026-06-16 11:57:41,866][src.utils.utils][INFO] - [rank: 0] Enforcing tags! <cfg.extras.enforce_tags=True>
+[2026-06-16 11:57:41,871][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! <cfg.extras.print_config=True>
+[2026-06-16 11:57:42,145][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic.
+[2026-06-16 11:57:42,153][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule <src.data.loading.datamodules.sequence_datamodule.SequenceDataModule>
+[2026-06-16 11:57:42,240][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model <src.models.modules.semantic_id.tiger_generation_model.SemanticIDEncoderDecoder>
+[2026-06-16 11:57:42,491][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks...
+[2026-06-16 11:57:42,492][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback <lightning.pytorch.callbacks.ModelCheckpoint>
+[2026-06-16 11:57:42,495][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback <lightning.pytorch.callbacks.EarlyStopping>
+[2026-06-16 11:57:42,498][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback <lightning.pytorch.callbacks.RichModelSummary>
+[2026-06-16 11:57:42,498][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback <src.utils.restart_job.RestartAndLoadCheckpointCallback>
+[2026-06-16 11:57:42,500][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json
+[2026-06-16 11:57:42,501][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json. Creating empty metadata.
+[2026-06-16 11:57:42,501][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.501360', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/beauty_rkmeans', '++should_skip_retry=True']}
+[2026-06-16 11:57:42,501][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers...
+[2026-06-16 11:57:42,502][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger <lightning.pytorch.loggers.csv_logs.CSVLogger>
+[2026-06-16 11:57:42,503][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer <lightning.pytorch.trainer.Trainer>
+[2026-06-16 11:57:42,505][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json
+[2026-06-16 11:57:42,507][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json
+[2026-06-16 11:57:42,508][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json
+[2026-06-16 11:57:42,554][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters!
+[2026-06-16 11:57:42,950][__main__][INFO] - [rank: 0] Starting training!
+[2026-06-16 11:57:44,675][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/beauty_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:42.501360', 'restarts': [], 'current_run': 0, 'used_ports': ['42809'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/beauty', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/beauty/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/beauty_rkmeans', '++should_skip_retry=True']}
+[2026-06-16 17:02:05,020][__main__][INFO] - [rank: 0] Starting testing!
+[2026-06-16 17:04:12,741][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/L/beauty_rkmeans/checkpoints/checkpoint_epoch=000_step=004800.ckpt
+[2026-06-16 17:04:12,767][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(7.5031), 'train/loss_step': tensor(4.6689), 'val/loss': tensor(9.5383), 'val/ndcg@5': tensor(0.0393), 'val/ndcg@10': tensor(0.0470), 'val/recall@5': tensor(0.0569), 'val/recall@10': tensor(0.0809), 'train/loss_epoch': tensor(7.5031), 'test/loss': tensor(9.8078), 'test/ndcg@5': tensor(0.0304), 'test/ndcg@10': tensor(0.0365), 'test/recall@5': tensor(0.0449), 'test/recall@10': tensor(0.0640)}
+[2026-06-16 17:04:21,722][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully.

jobs/logs/reproducing/L/sports/train.log

@@ -0,0 +1,30 @@
+[2026-06-16 11:57:35,632][src.utils.utils][INFO] - [rank: 0] Enforcing tags! <cfg.extras.enforce_tags=True>
+[2026-06-16 11:57:35,642][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! <cfg.extras.print_config=True>
+[2026-06-16 11:57:35,922][src.utils.restart_job][INFO] - [rank: 0] Using metadata dir: /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata
+[2026-06-16 11:57:35,922][src.utils.restart_job][INFO] - [rank: 0] Starting job (attempt 1/4): /home/scur1219/.conda/envs/RecSys/bin/python /gpfs/home6/scur1219/GRID/src/train.py experiment=tiger_train_flat data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt num_hierarchies=4 hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/sports_rkmeans ++should_skip_retry=True
+[2026-06-16 11:57:44,389][src.utils.utils][INFO] - [rank: 0] Enforcing tags! <cfg.extras.enforce_tags=True>
+[2026-06-16 11:57:44,394][src.utils.utils][INFO] - [rank: 0] Printing config tree with Rich! <cfg.extras.print_config=True>
+[2026-06-16 11:57:44,670][src.utils.restart_job][INFO] - [rank: 0] should_skip_retry set to True. Skipping retry logic.
+[2026-06-16 11:57:44,679][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating datamodule <src.data.loading.datamodules.sequence_datamodule.SequenceDataModule>
+[2026-06-16 11:57:44,801][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating model <src.models.modules.semantic_id.tiger_generation_model.SemanticIDEncoderDecoder>
+[2026-06-16 11:57:45,077][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating callbacks...
+[2026-06-16 11:57:45,077][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback <lightning.pytorch.callbacks.ModelCheckpoint>
+[2026-06-16 11:57:45,081][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback <lightning.pytorch.callbacks.EarlyStopping>
+[2026-06-16 11:57:45,083][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback <lightning.pytorch.callbacks.RichModelSummary>
+[2026-06-16 11:57:45,084][src.utils.instantiators][INFO] - [rank: 0] Instantiating callback <src.utils.restart_job.RestartAndLoadCheckpointCallback>
+[2026-06-16 11:57:45,086][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json
+[2026-06-16 11:57:45,086][src.utils.restart_job_utils][WARNING] - [rank: 0] Metadata file not found at /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json. Creating empty metadata.
+[2026-06-16 11:57:45,086][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:45.086844', 'restarts': [], 'current_run': 0, 'used_ports': [], 'world_size': 0, 'node_rank': 0, 'master_addr': '', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/sports_rkmeans', '++should_skip_retry=True']}
+[2026-06-16 11:57:45,087][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating loggers...
+[2026-06-16 11:57:45,087][src.utils.instantiators][INFO] - [rank: 0] Instantiating logger <lightning.pytorch.loggers.csv_logs.CSVLogger>
+[2026-06-16 11:57:45,089][src.utils.launcher_utils][INFO] - [rank: 0] Instantiating trainer <lightning.pytorch.trainer.Trainer>
+[2026-06-16 11:57:45,091][src.utils.restart_job_utils][INFO] - [rank: 0] Trying to load metadata from /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json
+[2026-06-16 11:57:45,093][src.utils.restart_job_utils][INFO] - [rank: 0] Metadata loaded successfully from /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json
+[2026-06-16 11:57:45,093][src.utils.restart_job_utils][INFO] - [rank: 0] Retrieved current_run: 0 from metadata /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json
+[2026-06-16 11:57:45,141][src.utils.launcher_utils][INFO] - [rank: 0] Logging hyperparameters!
+[2026-06-16 11:57:45,571][__main__][INFO] - [rank: 0] Starting training!
+[2026-06-16 11:57:47,411][src.utils.restart_job_utils][INFO] - [rank: 0] Saving metadata to /scratch-shared/scur1219/results/tiger/L/sports_rkmeans/metadata/restart_metadata.json. {'start_time': '2026-06-16T11:57:45.086844', 'restarts': [], 'current_run': 0, 'used_ports': ['47843'], 'world_size': 1, 'node_rank': 0, 'master_addr': '127.0.0.1', 'original_args': ['/gpfs/home6/scur1219/GRID/src/train.py', 'experiment=tiger_train_flat', 'data_dir=/projects/prjs2120/groups/group_08/data/amazon_data/sports', 'semantic_id_path=/scratch-shared/scur1219/results/sid_rkmeans/L/sports/rkmeans_inference/pickle/merged_predictions_tensor.pt', 'num_hierarchies=4', 'hydra.run.dir=/scratch-shared/scur1219/results/tiger/L/sports_rkmeans', '++should_skip_retry=True']}
+[2026-06-16 17:28:49,970][__main__][INFO] - [rank: 0] Starting testing!
+[2026-06-16 17:32:11,383][__main__][INFO] - [rank: 0] Best ckpt path: /gpfs/scratch1/shared/scur1219/results/tiger/L/sports_rkmeans/checkpoints/checkpoint_epoch=000_step=004100.ckpt
+[2026-06-16 17:32:11,391][__main__][INFO] - [rank: 0] Metrics: {'train/loss': tensor(9.3024), 'train/loss_step': tensor(8.0710), 'val/loss': tensor(9.2528), 'val/ndcg@5': tensor(0.0193), 'val/ndcg@10': tensor(0.0241), 'val/recall@5': tensor(0.0294), 'val/recall@10': tensor(0.0440), 'train/loss_epoch': tensor(9.3024), 'test/loss': tensor(9.5419), 'test/ndcg@5': tensor(0.0147), 'test/ndcg@10': tensor(0.0182), 'test/recall@5': tensor(0.0223), 'test/recall@10': tensor(0.0331)}
+[2026-06-16 17:32:15,236][src.utils.restart_job][INFO] - [rank: 0] Job finished successfully.