Decoder only

README.md

@@ -105,6 +105,45 @@ python -m src.inference experiment=tiger_inference_flat \
     num_hierarchies=4 \ # Please note that we add 1 for num_hierarchies because in the previous step we appended one additional digit to de-duplicate the semantic IDs we generate.
 ```
 
+### Decoder-Only Model (Extension)
+
+In addition to the default encoder-decoder (T5-style) recommender, this repo
+provides a **decoder-only (GPT-style)** generative recommender,
+`SemanticIDDecoderOnly`. It is enabled entirely through the config (no code
+changes are needed) by selecting the decoder-only experiment instead of the
+default `tiger_*_flat` one:
+
+```bash
+# Train the decoder-only model
+python -m src.train experiment=tiger_decoder_only_train_flat \
+    data_dir=data/amazon_data/beauty \
+    semantic_id_path=<output_path_from_step_3>/pickle/merged_predictions_tensor.pt \
+    num_hierarchies=4 \
+    model.loss_on_all_positions=false   # see loss toggle below
+```
+
+```bash
+# Generate recommendations with the decoder-only model
+python -m src.inference experiment=tiger_decoder_only_inference_flat \
+    data_dir=data/amazon_data/beauty \
+    semantic_id_path=<output_path_from_step_3>/pickle/merged_predictions_tensor.pt \
+    ckpt_path=<the_checkpoint_you_just_get_above> \
+    num_hierarchies=4
+```
+
+**Loss objective toggle.** The decoder-only model exposes a next-token
+prediction at every position, so it supports two training objectives via the
+`model.loss_on_all_positions` flag (default `true` in the experiment config):
+
+- `model.loss_on_all_positions=false`: apply the loss only on the final target
+  item (**loss-last**), matching the encoder-decoder TIGER objective.
+- `model.loss_on_all_positions=true`: apply the full causal next-token loss on
+  **all positions** (**loss-all**).
+
+All other knobs (Semantic IDs, model dimensions, optimizer, beam search) are
+shared with the encoder-decoder baseline, so switching the `experiment` value is
+sufficient to compare architectures under identical settings.
+
 ## Supported Models:
 
 ### Semantic ID:
@@ -115,7 +154,8 @@ python -m src.inference experiment=tiger_inference_flat \
 
 ### Generative Recommendation:
 
-1. TIGER [1]
+1. TIGER [1] (encoder-decoder)
+2. Decoder-only (GPT-style) variant with selectable loss-last / loss-all objective
 
 ## 📚 Citation
 

configs/experiment/tiger_inference_flat.yaml

@@ -8,17 +8,23 @@ sequence_length: 120
 
 model:
   huggingface_model:
-    _target_: transformers.T5EncoderModel
+    _target_: transformers.models.t5.modeling_t5.T5Stack
     config:
-      _target_: transformers.T5Config
+      _target_: transformers.models.t5.configuration_t5.T5Config
       vocab_size: 256
       d_model: 128
       num_heads: 6
       dropout_rate: 0.15
       d_ff: 1024
       d_kv: 64
       num_layers: 4
-  _target_: src.models.modules.semantic_id.tiger_generation_model.SemanticIDEncoderDecoder
+      is_decoder: true
+      is_encoder_decoder: false
+    embed_tokens:
+      _target_: torch.nn.Embedding
+      num_embeddings: ${model.huggingface_model.config.vocab_size}
+      embedding_dim: ${model.huggingface_model.config.d_model}
+  _target_: src.models.modules.semantic_id.tiger_generation_model.SemanticIDDecoderOnly
   feature_to_model_input_map:
     sequence_data: input_ids
     user_id: user_id
@@ -30,27 +36,12 @@ model:
   evaluator: null
   weight_tying: true
   compile: false
-  decoder:
-    _target_: transformers.models.t5.modeling_t5.T5Stack
-    config:
-      _target_: transformers.models.t5.configuration_t5.T5Config
-      vocab_size: ${model.huggingface_model.config.vocab_size}
-      d_model: ${model.huggingface_model.config.d_model}
-      num_heads: ${model.huggingface_model.config.num_heads}
-      dropout_rate: 0.15
-      d_ff: ${model.huggingface_model.config.d_ff}
-      d_kv: ${model.huggingface_model.config.d_kv}
-      num_layers: 4
-      is_decoder: true
-      is_encoder_decoder: false
-    embed_tokens:
-      _target_: torch.nn.Embedding
-      num_embeddings: ${model.huggingface_model.config.vocab_size}
-      embedding_dim: ${model.huggingface_model.config.d_model}
+  decoder: null
   num_hierarchies: ${num_hierarchies}
   num_user_bins: null
   codebooks: ${data_loading.predict_dataloader_config.dataloader.dataset_config.semantic_id_map.sequence_data}
   mlp_layers: 2
+  loss_on_all_positions: true
   top_k_for_generation: 10
 task_name: inference
 id: ${now:%Y-%m-%d}/${now:%H-%M-%S}
@@ -218,4 +209,3 @@ extras:
   enforce_tags: true
   print_config_warnings: true
   print_config: true
-

configs/experiment/tiger_train_flat.yaml

@@ -148,191 +148,3 @@ data_loading:
       assign_files_by_size: true
       oov_token: ${data_loading.train_dataloader_config.dataloader.oov_token}
       masking_token: ${data_loading.train_dataloader_config.dataloader.masking_token}
-      sequence_length: ${data_loading.train_dataloader_config.dataloader.sequence_length}
-      padding_token: ${data_loading.train_dataloader_config.dataloader.padding_token}
-      drop_last: false
-      persistent_workers: false
-      collate_fn:
-        _target_: src.data.loading.components.collate_functions.collate_fn_train
-        _partial_: true
-        sequence_length: ${data_loading.train_dataloader_config.dataloader.sequence_length}
-        padding_token: ${data_loading.train_dataloader_config.dataloader.padding_token}
-      dataset_config: ${data_loading.train_dataloader_config.dataloader.dataset_config}
-      pin_memory: false
-  test_dataloader_config:
-    dataloader:
-      _target_: src.data.loading.components.interfaces.SequenceDataloaderConfig
-      dataset_class:
-        _target_: src.data.loading.components.dataloading.UnboundedSequenceIterable
-        _partial_: true
-      data_folder: ${paths.data_dir}/testing
-      should_shuffle_rows: false
-      labels:
-        sequence_data:
-          transform:
-            _target_: src.data.loading.components.label_function.NextKTokenMasking
-            next_k: ${model.num_hierarchies}
-      batch_size_per_device: 8
-      num_workers: 8
-      timeout: 60
-      assign_files_by_size: true
-      oov_token: ${data_loading.train_dataloader_config.dataloader.oov_token}
-      masking_token: ${data_loading.train_dataloader_config.dataloader.masking_token}
-      sequence_length: ${data_loading.train_dataloader_config.dataloader.sequence_length}
-      padding_token: ${data_loading.train_dataloader_config.dataloader.padding_token}
-      drop_last: false
-      persistent_workers: false
-      collate_fn:
-        _target_: src.data.loading.components.collate_functions.collate_fn_train
-        _partial_: true
-        sequence_length: ${data_loading.train_dataloader_config.dataloader.sequence_length}
-        padding_token: ${data_loading.train_dataloader_config.dataloader.padding_token}
-      dataset_config: ${data_loading.train_dataloader_config.dataloader.dataset_config}
-      pin_memory: false
-  datamodule:
-    _target_: src.data.loading.datamodules.sequence_datamodule.SequenceDataModule
-    train_dataloader_config: ${..train_dataloader_config.dataloader}
-    val_dataloader_config: ${..val_dataloader_config.dataloader}
-    test_dataloader_config: ${..test_dataloader_config.dataloader}
-model:
-  huggingface_model:
-    _target_: transformers.T5EncoderModel
-    config:
-      _target_: transformers.T5Config
-      vocab_size: 256
-      d_model: 128
-      num_heads: 6
-      dropout_rate: 0.15
-      d_ff: 1024
-      d_kv: 64
-      num_layers: 4
-  _target_: src.models.modules.semantic_id.tiger_generation_model.SemanticIDEncoderDecoder
-  feature_to_model_input_map:
-    sequence_data: input_ids
-    user_id: user_id
-  postprocessor: null
-  aggregator: null
-  loss_function: ${loss.loss_function}
-  optimizer: ${optim.optimizer}
-  scheduler: ${optim.scheduler}
-  evaluator: ${eval.evaluator}
-  weight_tying: true
-  compile: false
-  decoder:
-    _target_: transformers.models.t5.modeling_t5.T5Stack
-    config:
-      _target_: transformers.models.t5.configuration_t5.T5Config
-      vocab_size: ${model.huggingface_model.config.vocab_size}
-      d_model: ${model.huggingface_model.config.d_model}
-      num_heads: ${model.huggingface_model.config.num_heads}
-      dropout_rate: 0.15
-      d_ff: ${model.huggingface_model.config.d_ff}
-      d_kv: ${model.huggingface_model.config.d_kv}
-      num_layers: 4
-      is_decoder: true
-      is_encoder_decoder: false
-    embed_tokens:
-      _target_: torch.nn.Embedding
-      num_embeddings: ${model.huggingface_model.config.vocab_size}
-      embedding_dim: ${model.huggingface_model.config.d_model}
-  num_hierarchies: ${num_hierarchies}
-  num_user_bins: null
-  codebooks: ${data_loading.train_dataloader_config.dataloader.dataset_config.semantic_id_map.sequence_data}
-  mlp_layers: 2
-callbacks:
-  model_checkpoint:
-    _target_: lightning.pytorch.callbacks.ModelCheckpoint
-    dirpath: ${paths.output_dir}/checkpoints
-    filename: checkpoint_{epoch:03d}_{step:06d}
-    monitor: val/recall@5
-    verbose: true
-    save_last: null
-    save_top_k: 1
-    mode: max
-    auto_insert_metric_name: true
-    save_weights_only: false
-    every_n_train_steps: null
-    train_time_interval: null
-    every_n_epochs: null
-    save_on_train_epoch_end: false
-  early_stopping:
-    _target_: lightning.pytorch.callbacks.EarlyStopping
-    monitor: ${callbacks.model_checkpoint.monitor}
-    min_delta: 0.0
-    patience: 10
-    verbose: true
-    mode: ${callbacks.model_checkpoint.mode}
-    strict: true
-    check_finite: true
-    stopping_threshold: null
-    divergence_threshold: null
-    check_on_train_epoch_end: false
-  model_summary:
-    _target_: lightning.pytorch.callbacks.RichModelSummary
-    max_depth: -1
-  restart_job:
-    _target_: src.utils.restart_job.RestartAndLoadCheckpointCallback
-    metadata_dir: ${paths.metadata_dir}
-logger:
-  csv:
-    _target_: lightning.pytorch.loggers.csv_logs.CSVLogger
-    save_dir: ${paths.output_dir}
-    name: csv/
-    prefix: ''
-trainer:
-  _target_: lightning.pytorch.trainer.Trainer
-  default_root_dir: ${paths.output_dir}
-  min_steps: 1
-  max_steps: 320000
-  max_epochs: 10
-  accelerator: gpu
-  devices: -1
-  num_nodes: 1
-  precision: 32-true
-  log_every_n_steps: 100
-  val_check_interval: 1600
-  deterministic: false
-  accumulate_grad_batches: 16
-  profiler:
-    _target_: lightning.pytorch.profilers.PassThroughProfiler
-  strategy: ddp
-  sync_batchnorm: true
-  num_sanity_val_steps: 0
-  min_epochs: 0
-paths:
-  root_dir: .
-  data_dir: ${data_dir}
-  log_dir: ${paths.root_dir}/logs
-  output_dir: ${hydra:runtime.output_dir}
-  work_dir: ${hydra:runtime.cwd}
-  profile_dir: ${hydra:run.dir}/profile_output
-  metadata_dir: ${paths.output_dir}/metadata
-extras:
-  ignore_warnings: false
-  enforce_tags: true
-  print_config_warnings: true
-  print_config: true
-loss:
-  loss_function:
-    _target_: torch.nn.CrossEntropyLoss
-optim:
-  optimizer:
-    _target_: torch.optim.Adam
-    _partial_: true
-    lr: 0.001
-    weight_decay: 0.0001
-  scheduler: null
-eval:
-  evaluator:
-    _target_: src.components.eval_metrics.SIDRetrievalEvaluator
-    top_k_list:
-    - 5
-    - 10
-    metrics:
-      ndcg:
-        _target_: src.components.eval_metrics.NDCG
-        _partial_: true
-      recall:
-        _target_: src.components.eval_metrics.Recall
-        _partial_: true
-

jobs/decoder_only_inference.job

@@ -0,0 +1,46 @@
+#!/bin/bash
+#SBATCH --job-name=test_dec_only
+#SBATCH --output=logs/test_dec_only_%j.out
+#SBATCH --error=logs/test_dec_only_%j.err
+#SBATCH --partition=gpu_mig
+#SBATCH --gpus=1
+#SBATCH --time=00:15:00
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+
+module purge
+module load 2023
+module load Anaconda3/2023.07-2
+module load CUDA/12.4.0
+
+source $(conda info --base)/etc/profile.d/conda.sh
+conda activate grid
+
+# Usage: First dataset (beauty, toys, sports), then 
+# model: (grid_decoder_only_all  grid_decoder_only_all_noaug  grid_decoder_only_last  grid_decoder_only_last_noaug)
+
+DATASET=$1
+MODEL=$2
+
+export HYDRA_FULL_ERROR=1
+
+export OMP_NUM_THREADS=8
+
+cd $HOME/GRID
+
+BASE=/projects/prjs2120/groups/group_08
+
+# We can change to grid_decoder_only_last (best performing loss last),  grid_decoder_only_all_noaug (best performing loss all)
+CKPT=$(ls /projects/prjs2120/groups/group_08/results/decoder_only/${MODEL}/${DATASET}/checkpoints/checkpoint_epoch=*.ckpt | head -1)
+
+mkdir -p $HOME/GRID/${MODEL}/${DATASET}/outputs
+
+python -m src.inference experiment=tiger_inference_flat \
+    data_dir=$BASE/data/amazon_data/$DATASET \
+    semantic_id_path=$BASE/results/sid_rkmeans/${DATASET}/rkmeans_inference/pickle/merged_predictions_tensor.pt \
+    ckpt_path="'$CKPT'" \
+    num_hierarchies=4 \
+    hydra.run.dir=$HOME/GRID/${MODEL}/${DATASET} \
+    paths.output_dir=$HOME/GRID/${MODEL}/${DATASET}/outputs \
+    ++should_skip_retry=True