Skip to content

change download url #320

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
niconiconi merged 1 commit into from
Feb 6, 2026
+30 −106
Merged

change download url #320

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 30 additions & 106 deletions gkr/src/utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use std::fs;
use std::process::Command;

const DATA_PREFIX: &str = "data/";
const URL_PREFIX: &str = "https://pub-3be2514d8cd1470691b87e515984f903.r2.dev";

// circuit for repeating Keccak for 2 times
pub const KECCAK_M31_CIRCUIT: &str = "data/circuit_m31.txt";
Expand All @@ -26,133 +27,56 @@ pub const KECCAK_M31_PROOF: &str = "data/proof_m31.txt";
pub const KECCAK_GF2_PROOF: &str = "data/proof_gf2.txt";
pub const KECCAK_BN254_PROOF: &str = "data/proof_bn254.txt";

// GitHub Release tag for test data
// Update this tag when regenerating test data
pub const TESTDATA_TAG: &str = "testdata-v1";
pub const GITHUB_RELEASE_BASE: &str = "https://github.com/PolyhedraZK/Expander/releases/download";

// URL for Keccak circuit repeated for 2 times
pub const KECCAK_CIRCUIT_M31_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/circuit_m31.txt");
pub const KECCAK_CIRCUIT_GF2_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/circuit_gf2.txt");
pub const KECCAK_CIRCUIT_BN254_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/circuit_bn254.txt");
pub const KECCAK_CIRCUIT_GOLDILOCKS_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/circuit_goldilocks.txt");
pub const KECCAK_CIRCUIT_BABYBEAR_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/circuit_babybear.txt");

pub const KECCAK_WITNESS_M31_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/witness_m31.txt");
pub const KECCAK_WITNESS_GF2_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/witness_gf2.txt");
pub const KECCAK_WITNESS_BN254_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/witness_bn254.txt");
pub const KECCAK_WITNESS_GOLDILOCKS_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/witness_goldilocks.txt");
pub const KECCAK_WITNESS_BABYBEAR_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/witness_babybear.txt");

pub const KECCAK_WITNESS_M31_MPI2_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/witness_m31_mpi_2.txt");
pub const KECCAK_WITNESS_GF2_MPI2_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/witness_gf2_mpi_2.txt");
pub const KECCAK_WITNESS_BN254_MPI2_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/witness_bn254_mpi_2.txt");
pub const KECCAK_WITNESS_GOLDILOCKS_MPI2_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/witness_goldilocks_mpi_2.txt");
pub const KECCAK_WITNESS_BABYBEAR_MPI2_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/witness_babybear_mpi_2.txt");

// circuit for repeating Poseidon for 120 times
pub const POSEIDON_M31_CIRCUIT: &str = "data/poseidon_120_circuit_m31.txt";
// circuit for repeating Poseidon for 120 times
pub const POSEIDON_M31_WITNESS: &str = "data/poseidon_120_witness_m31.txt";
// // circuit for repeating Poseidon for 120 times
// pub const POSEIDON_BN254_CIRCUIT: &str = "data/poseidon_120_circuit_bn254.txt";
// // circuit for repeating Poseidon for 120 times
// pub const POSEIDON_BN254_WITNESS: &str = "data/poseidon_120_witness_bn254.txt";

// URL for Poseidon circuit repeated for 120 times
pub const POSEIDON_CIRCUIT_M31_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/poseidon_120_circuit_m31.txt");
// URL for Poseidon circuit repeated for 120 times
pub const POSEIDON_WITNESS_M31_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/poseidon_120_witness_m31.txt");
// // URL for Poseidon circuit repeated for 120 times
// pub const POSEIDON_CIRCUIT_BN254_URL: &str =
// concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/poseidon_120_circuit_bn254.txt");
// // URL for Poseidon circuit repeated for 120 times
// pub const POSEIDON_WITNESS_BN254_URL: &str =
// concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/poseidon_120_witness_bn254.txt");

// URL for proofs
pub const KECCAK_M31_PROOF_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/proof_m31.txt");
pub const KECCAK_GF2_PROOF_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/proof_gf2.txt");
pub const KECCAK_BN254_PROOF_URL: &str =
concat!("https://github.com/PolyhedraZK/Expander/releases/download/testdata-v1/proof_bn254.txt");

// NOTE(Hang 08/23/24):
// CI process is unhappy about reqwest as a dependency,
// so we use wget as a backup option.
// NOTE: -L flag is needed to follow redirects (required for GitHub releases)
fn download_and_store(url: &str, file: &str) {
let download = Command::new("bash")
.arg("-c")
.arg(format!("wget -L {url} -O {file}"))
.output()
.expect("Failed to execute wget");

if !download.status.success() {
eprintln!("Warning: Failed to download {url} to {file}");
fn download_and_store(url_path: &str, file: &str) {
// Skip download if file already exists
if std::path::Path::new(file).exists() {
println!("File already exists, skipping download: {}", file);
return;
}
}

fn download_and_store_required(url: &str, file: &str) {
let url = format!("{}{}", URL_PREFIX, url_path);
let download = Command::new("bash")
.arg("-c")
.arg(format!("wget -L {url} -O {file}"))
.arg(format!("wget {} -O {}", url, file))
Comment on lines 45 to +47

@gemini-code-assist gemini-code-assist Bot Feb 5, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Using bash -c with a formatted string to execute commands can be risky as it can lead to shell injection vulnerabilities if the arguments are not properly sanitized. Although the url and file variables seem to be from controlled sources in this case, it's a best practice to avoid invoking a shell when possible. Passing arguments directly to Command::new("wget") is safer and more robust.

Suggested change
let download = Command::new("bash")
.arg("-c")
.arg(format!("wget -L {url} -O {file}"))
.arg(format!("wget {} -O {}", url, file))
let download = Command::new("wget").arg(&url).arg("-O").arg(file)

.output()
.expect("Failed to execute wget");
.expect("Failed to download circuit");

assert!(download.status.success(), "Required file download failed: {url}");
assert!(download.status.success(), "Circuit download failure: {}", url)
}
Comment on lines +37 to 52

@gemini-code-assist gemini-code-assist Bot Feb 5, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The new download_and_store function consolidates the logic of the previous download_and_store and download_and_store_required functions. However, this new implementation will panic on any download failure, which was previously the behavior only for required files. Files that were considered optional (like Goldilocks and BabyBear circuits) will now cause the setup to fail if they cannot be downloaded. If all files are now guaranteed to be available at the new URL, this change is acceptable. Otherwise, you might want to consider re-introducing a non-panicking download variant for optional files.


pub fn dev_env_data_setup() {
fs::create_dir_all(DATA_PREFIX).unwrap();

// Required keccak circuits (GF2, M31, BN254)
download_and_store_required(KECCAK_CIRCUIT_M31_URL, KECCAK_M31_CIRCUIT);
download_and_store_required(KECCAK_CIRCUIT_GF2_URL, KECCAK_GF2_CIRCUIT);
download_and_store_required(KECCAK_CIRCUIT_BN254_URL, KECCAK_BN254_CIRCUIT);

// Optional keccak circuits (Goldilocks, BabyBear) - may not be available
download_and_store(KECCAK_CIRCUIT_GOLDILOCKS_URL, KECCAK_GOLDILOCKS_CIRCUIT);
download_and_store(KECCAK_CIRCUIT_BABYBEAR_URL, KECCAK_BABYBEAR_CIRCUIT);

// Required witnesses
download_and_store_required(KECCAK_WITNESS_M31_URL, KECCAK_M31_WITNESS);
download_and_store_required(KECCAK_WITNESS_GF2_URL, KECCAK_GF2_WITNESS);
download_and_store_required(KECCAK_WITNESS_BN254_URL, KECCAK_BN254_WITNESS);
// keccak circuit
download_and_store("/keccak-ci/serialization-v6/circuit_m31.txt", KECCAK_M31_CIRCUIT);
download_and_store("/keccak-ci/serialization-v6/circuit_gf2.txt", KECCAK_GF2_CIRCUIT);
download_and_store("/keccak-ci/serialization-v6/circuit_bn254.txt", KECCAK_BN254_CIRCUIT);
download_and_store("/keccak-ci/serialization-v6/circuit_goldilocks.txt", KECCAK_GOLDILOCKS_CIRCUIT);
download_and_store("/keccak-ci/serialization-v6/circuit_babybear.txt", KECCAK_BABYBEAR_CIRCUIT);

// Optional witnesses (Goldilocks, BabyBear)
download_and_store(KECCAK_WITNESS_GOLDILOCKS_URL, KECCAK_GOLDILOCKS_WITNESS);
download_and_store(KECCAK_WITNESS_BABYBEAR_URL, KECCAK_BABYBEAR_WITNESS);
download_and_store("/keccak-ci/serialization-v6/witness_m31.txt", KECCAK_M31_WITNESS);
download_and_store("/keccak-ci/serialization-v6/witness_gf2.txt", KECCAK_GF2_WITNESS);
download_and_store("/keccak-ci/serialization-v6/witness_bn254.txt", KECCAK_BN254_WITNESS);
download_and_store("/keccak-ci/serialization-v6/witness_goldilocks.txt", KECCAK_GOLDILOCKS_WITNESS);
download_and_store("/keccak-ci/serialization-v6/witness_babybear.txt", KECCAK_BABYBEAR_WITNESS);

// Proofs
download_and_store_required(KECCAK_M31_PROOF_URL, KECCAK_M31_PROOF);
download_and_store_required(KECCAK_GF2_PROOF_URL, KECCAK_GF2_PROOF);
download_and_store_required(KECCAK_BN254_PROOF_URL, KECCAK_BN254_PROOF);
download_and_store("/keccak-ci/serialization-v6/proof_m31.txt", KECCAK_M31_PROOF);
download_and_store("/keccak-ci/serialization-v6/proof_gf2.txt", KECCAK_GF2_PROOF);
download_and_store("/keccak-ci/serialization-v6/proof_bn254.txt", KECCAK_BN254_PROOF);

// MPI witnesses - optional (may not be in release)
download_and_store(KECCAK_WITNESS_M31_MPI2_URL, KECCAK_M31_MPI2_WITNESS);
download_and_store(KECCAK_WITNESS_GF2_MPI2_URL, KECCAK_GF2_MPI2_WITNESS);
download_and_store(KECCAK_WITNESS_BN254_MPI2_URL, KECCAK_BN254_MPI2_WITNESS);
// keccak circuit with MPI
download_and_store("/keccak-ci/serialization-v6/witness_m31_mpi_2.txt", KECCAK_M31_MPI2_WITNESS);
download_and_store("/keccak-ci/serialization-v6/witness_gf2_mpi_2.txt", KECCAK_GF2_MPI2_WITNESS);
download_and_store("/keccak-ci/serialization-v6/witness_bn254_mpi_2.txt", KECCAK_BN254_MPI2_WITNESS);

// poseidon circuit
download_and_store_required(POSEIDON_CIRCUIT_M31_URL, POSEIDON_M31_CIRCUIT);
download_and_store_required(POSEIDON_WITNESS_M31_URL, POSEIDON_M31_WITNESS);
download_and_store("/poseidon-ci/poseidon_120_circuit_m31.txt", POSEIDON_M31_CIRCUIT);
download_and_store("/poseidon-ci/poseidon_120_witness_m31.txt", POSEIDON_M31_WITNESS);
}
Loading