Run experiments

llm_fo_fine_tune_main.py

@@ -80,15 +80,15 @@ def inf_loader(dl):
             torch.cuda.empty_cache()
     print(f"Start, Accuracy: {acc.avg:.4f}")
 
-    num_epochs = 20
     train_loader = train_loaders[0]
     model.train()
+    eval_iterations = 20
+    iterations = 1000
     total_loss = 0.0
     inf_train_loader = inf_loader(train_loader)
-    eval_iterations = 200
     train_losses = []
     eval_accs = []
-    for i in tqdm(range(10000)):
+    for i in tqdm(range(iterations)):
         batch_input_dict, batch_output_tensor = next(inf_train_loader)
         batch_input_dict = batch_input_dict.to("cuda")
         batch_output_tensor = batch_output_tensor.to("cuda")

run_experiments.sh

@@ -0,0 +1,2 @@
+uv run decomfl_main.py --large-model=smollm3-3b --dataset=sst2 --iterations=3000 --train-batch-size=32 --test-batch-size=32 --eval-iterations=25 --num-clients=6 --num-sample-clients=2 --local-update-steps=1 --num-pert=5 --lr=1e-6 --mu=1e-3 --grad-estimate-method=rge-forward --no-optim  --log-to-tensorboard=decomfl-sst2-smollm3-3b
+uv run decomfl_main.py --dataset=sst2 --eval-iterations=25 --large-model=smollm3-3b --model-dtype=float32 --seed=365 --iterations=3000 --train-batch-size=32 --test-batch-size=32 --no-iid --dirichlet-alpha=1 --num-clients=6 --num-sample-clients=2 --local-update-steps=1 --num-pert=5 --lr=0.00001 --momentum=0 --grad-estimate-method=rge-forward --mu=0.001 --estimator-type=adam_forward --k_update_strategy=last_local_update --hessian-smooth=0.95 --no-optim --log-to-tensorboard=Hessian-sst2-smollm3-3b