diff --git a/llm_fo_fine_tune_main.py b/llm_fo_fine_tune_main.py index 17e45b9..4d37548 100644 --- a/llm_fo_fine_tune_main.py +++ b/llm_fo_fine_tune_main.py @@ -80,15 +80,15 @@ def inf_loader(dl): torch.cuda.empty_cache() print(f"Start, Accuracy: {acc.avg:.4f}") - num_epochs = 20 train_loader = train_loaders[0] model.train() + eval_iterations = 20 + iterations = 1000 total_loss = 0.0 inf_train_loader = inf_loader(train_loader) - eval_iterations = 200 train_losses = [] eval_accs = [] - for i in tqdm(range(10000)): + for i in tqdm(range(iterations)): batch_input_dict, batch_output_tensor = next(inf_train_loader) batch_input_dict = batch_input_dict.to("cuda") batch_output_tensor = batch_output_tensor.to("cuda") diff --git a/run_experiments.sh b/run_experiments.sh new file mode 100644 index 0000000..56be1c6 --- /dev/null +++ b/run_experiments.sh @@ -0,0 +1,2 @@ +uv run decomfl_main.py --large-model=smollm3-3b --dataset=sst2 --iterations=3000 --train-batch-size=32 --test-batch-size=32 --eval-iterations=25 --num-clients=6 --num-sample-clients=2 --local-update-steps=1 --num-pert=5 --lr=1e-6 --mu=1e-3 --grad-estimate-method=rge-forward --no-optim --log-to-tensorboard=decomfl-sst2-smollm3-3b +uv run decomfl_main.py --dataset=sst2 --eval-iterations=25 --large-model=smollm3-3b --model-dtype=float32 --seed=365 --iterations=3000 --train-batch-size=32 --test-batch-size=32 --no-iid --dirichlet-alpha=1 --num-clients=6 --num-sample-clients=2 --local-update-steps=1 --num-pert=5 --lr=0.00001 --momentum=0 --grad-estimate-method=rge-forward --mu=0.001 --estimator-type=adam_forward --k_update_strategy=last_local_update --hessian-smooth=0.95 --no-optim --log-to-tensorboard=Hessian-sst2-smollm3-3b