main_global.py

import os
import torch
import torch.optim.adam
from tools.utils import parse_args, load_yaml_config, merge_opts_to_config, \
    instantiate_from_config, seed_everything, create_mlab_eng, Logger
from pprint import pprint
from models.unet_dino_global import Unet
import numpy as np

def train(num_epochs, model, train_loader, optimizer, train_scheduler, criterion, device, cfg_drop_rate, save_ckpt_every, logger):
    for epoch_idx in range(num_epochs):
        losses = []

        model.train()
        batch_idx = 1
        for trajs, stims, dino_features, dino_patches, og_w, og_h, stim_names, subs in train_loader:
            print(f'Processing batch {batch_idx} out of {len(train_loader)} batches')
            # Zero gradients
            optimizer.zero_grad()

            # Move data to GPU
            trajs = trajs.to(device)

            # Sample random noise
            noise = torch.randn_like(trajs).to(device) # (batch_size, 2, seq_len)

            # Sample timestep
            t = torch.randint(0, len(train_scheduler.timesteps), (trajs.shape[0],)).to(device) # (batch_size,)

            # Add noise to images according to timestep
            noisy_trajs = train_scheduler.add_noise(trajs, noise, t) # (batch_size, 2, seq_len)

            # Run the correct model
            
            # Dropout rate for Dino Patches
            if np.random.uniform() < cfg_drop_rate:
                print('Training -- Dropping Dino Patches')
                dino_features = torch.zeros_like(dino_features)
                dino_features = dino_features.to(device)
            else:
                print('Training -- Not dropping Dino Patches')
                dino_features = dino_features.to(device)
 
            
            # Forward pass
            noise_pred = model(noisy_trajs, t, dino_features) # (batch_size, 2, seq_len)
            
            
            # Compute loss and save it to losses list for this epoch
            loss = criterion(noise_pred, noise)
            losses.append(loss.item())

            # Backpopagate
            loss.backward()
            optimizer.step()
            batch_idx += 1

        print(f'Epoch {epoch_idx + 1}/{num_epochs} - Loss: {np.mean(losses)}')

        # Save losses to a CSV file
        if logger is not None:
            save_losses(logger, epoch_idx, losses)
        
        # Save model and evaluate
  
        if epoch_idx % save_ckpt_every == 0 and epoch_idx != 0:
            save_model(logger, model, optimizer, epoch_idx)

        # # Should evaluate
        # if should_evaluate:
        #     if epoch_idx % evaluate_every == 0 and epoch_idx != 0:
        #         evaluate(epoch_idx)

def save_losses(logger, epoch_idx, losses):
    if logger is not None:
        losses_file = os.path.join(logger.logs_pth, 'losses.csv')
        with open(losses_file, 'a') as f:
            f.write(f'{epoch_idx + 1},{np.mean(losses)}\n')
                
def save_model(logger, model, optimizer, epoch_idx):
    if logger is not None:
        data = {
            'epoch': epoch_idx,
            'model_state_dict': model.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
        }

        torch.save(data, os.path.join(logger.checkpoints_pth, f'checkpoint_{epoch_idx}.pth'),)

def main():
    args = parse_args()

    config = load_yaml_config(args.config_file)
    config = merge_opts_to_config(config, args.opts)
    logger = Logger(args)
    logger.save_config(config, name='config.yaml')

    config['train_dataset']['params']['root'] = os.path.join(
        args.root_dir, 
        config['train_dataset']['params']['root']
    )
    config['train_dataset']['params']['stim_to_sub_pth'] = os.path.join(
        config['train_dataset']['params']['root'], 
        config['train_dataset']['params']['stim_to_sub_pth']
    )
    
    config['val_dataset']['params']['root'] = os.path.join(
        args.root_dir, 
        config['val_dataset']['params']['root']
    )
    config['val_dataset']['params']['stim_to_sub_pth'] = os.path.join(
        config['val_dataset']['params']['root'], 
        config['val_dataset']['params']['stim_to_sub_pth']
    )

    config['trainer']['params']['ground_truth_dir'] = os.path.join(
        config['train_dataset']['params']['root'], 
        config['trainer']['params']['ground_truth_dir']
    )

    print(config)
    

    logger.save_config(config, name='config_w_abs_paths.yaml')

    logger.save_git_commit()

    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    print(f'Using device: {device}')

    seed_everything(args.seed)

    # model = instantiate_from_config(config['model']).to(device)
    model = Unet(
        down_channels=[32, 64, 128, 256],
        mid_channels=[256, 256, 128],
        t_emb_dim=128,
        down_sample=[True, True, False],
        num_down_layers=2,
        num_mid_layers=2,
        num_up_layers=2
    ).to(device)

    # print trainable parameters number
    print(f'Trainable parameters: {sum(p.numel() for p in model.parameters() if p.requires_grad)}')
    optimizer = instantiate_from_config(
        config['optimizer'], 
        params=model.parameters()
    )

    ckpt_pth = args.ckpt_pth
    if ckpt_pth is not None:
        print(f'Loading checkpoint from {ckpt_pth}')
        checkpoint = torch.load(ckpt_pth, map_location='cpu')
        model.load_state_dict(checkpoint['model_state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])

    train_scheduler = instantiate_from_config(config['diff_train_scheduler'])
    
    train_dataset = instantiate_from_config(config['train_dataset'])
    train_loader = instantiate_from_config(
        config['train_loader'],
        dataset=train_dataset
    )

    

    criterion = instantiate_from_config(config['criterion'])

    if config['trainer']['params']['should_evaluate'] == True:
        val_scheduler = instantiate_from_config(config['diff_val_scheduler'])
        if config['diff_val_scheduler']['target'] == 'diffusers.schedulers.DDIMScheduler':
            val_scheduler.set_timesteps(config['diff_val_scheduler']['num_inference_steps'])
        val_dataset = instantiate_from_config(config['val_dataset'])
        val_loader = instantiate_from_config(
            config['val_loader'],
            dataset=val_dataset
        )

        matlab_eng = create_mlab_eng(
            os.path.join(config['train_dataset']['params']['root'], 'DatabaseCode'),
        )
    else:
        matlab_eng = None
        val_loader = None
        val_scheduler = None
        

    

    train(
        num_epochs=config['trainer']['params']['num_epochs'],
        model=model,
        train_loader=train_loader,
        optimizer=optimizer,
        train_scheduler=train_scheduler,
        criterion=criterion,
        device=device,
        cfg_drop_rate=config['trainer']['params']['cfg_drop_rate'],
        save_ckpt_every=config['trainer']['params']['save_ckpt_every'],
        logger=logger
    )
    
if __name__ == "__main__":
    main()