diff --git a/magma/magma.py b/magma/magma.py
index a2df2a2..df39a55 100644
--- a/magma/magma.py
+++ b/magma/magma.py
@@ -99,6 +99,7 @@ def __init__(self, config, device=None, init_weights=True):
         # freeze parameters
         if config.freeze_lm:
             for name, param in self.lm.named_parameters():  # freeze lm weights
+                param.requires_grad = False
                 if config.adapter_config and "adapter" in name:
                     param.requires_grad = True