diff --git a/.github/workflows/zombie-tests.yaml b/.github/workflows/zombie-tests.yaml index a488922df6..4431929b98 100644 --- a/.github/workflows/zombie-tests.yaml +++ b/.github/workflows/zombie-tests.yaml @@ -249,7 +249,9 @@ jobs: platform_config: ${{ fromJSON(needs.setup_matrix.outputs.platforms) }} scenario: [ { name: "0001 PVF preparation & execution time", test: "test-polkadot-functional-0001-parachains-pvf" }, + { name: "0001 PVF preparation & execution time with KAGOME minority", test: "test-polkadot-functional-0001-parachains-pvf-kagome-minority" }, { name: "0003 BEEFY voting and finality, test MMR proofs. Assumes Rococo sessions of 1 minute", test: "test-polkadot-functional-0003-beefy-and-mmr" }, + { name: "0003 BEEFY voting and finality, test MMR proofs. Assumes Rococo sessions of 1 minute with KAGOME minority", test: "test-polkadot-functional-0003-beefy-and-mmr-kagome-minority" }, { name: "0007 Test if disputes triggered on finalized blocks within scope always end as valid", test: "test-polkadot-functional-0007-dispute-freshly-finalized" }, { name: "0008 Test if disputes triggered on finalized blocks out of scope never get to be confirmed and concluded", test: "test-polkadot-functional-0008-dispute-old-finalized" }, { name: "0009 Approval voting coalescing does not lag finality", test: "test-polkadot-functional-0009-approval-voting-coalescing" }, diff --git a/zombienet/docker/Makefile.include/zombietests.mk b/zombienet/docker/Makefile.include/zombietests.mk index 7c0b935bfa..829cef6235 100644 --- a/zombienet/docker/Makefile.include/zombietests.mk +++ b/zombienet/docker/Makefile.include/zombietests.mk @@ -1,11 +1,16 @@ test-polkadot-functional-0001-parachains-pvf: $(call run_test, "kagome/zombienet/polkadot/functional/0001-parachains-pvf.zndsl") +test-polkadot-functional-0001-parachains-pvf-kagome-minority: + $(call run_test, "kagome/zombienet/polkadot/functional/0001-parachains-pvf-kagome-minority.zndsl") + test-polkadot-functional-0002-parachains-disputes: $(call run_test, "kagome/zombienet/polkadot/functional/0002-parachains-disputes.zndsl") test-polkadot-functional-0003-beefy-and-mmr: $(call run_test, "kagome/zombienet/polkadot/functional/0003-beefy-and-mmr.zndsl") +test-polkadot-functional-0003-beefy-and-mmr-kagome-minority: + $(call run_test, "kagome/zombienet/polkadot/functional/0003-beefy-and-mmr-kagome-minority.zndsl") test-polkadot-functional-0004-parachains-garbage-candidate: $(call run_test, "kagome/zombienet/polkadot/functional/0004-parachains-garbage-candidate.zndsl") diff --git a/zombienet/polkadot/functional/0001-parachains-pvf-kagome-minority.toml b/zombienet/polkadot/functional/0001-parachains-pvf-kagome-minority.toml new file mode 100644 index 0000000000..b0c4bc2f64 --- /dev/null +++ b/zombienet/polkadot/functional/0001-parachains-pvf-kagome-minority.toml @@ -0,0 +1,130 @@ +[settings] +timeout = 1000 + +[relaychain] +chain = "rococo-local" + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.nodes]] + name = "alice" + args = [ "--alice", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "bob" + args = [ "--bob", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "charlie" + args = [ "--charlie", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "dave" + args = [ "--dave", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "ferdie" + args = [ "--ferdie", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "eve" + args = [ "--eve", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "one" + command = "kagome" + prometheus_prefix = "kagome" + args = [ "--one", "--wasm-execution=Compiled", "-lparachain=debug", "-lnetwork=debug"] + + [[relaychain.nodes]] + name = "two" + command = "kagome" + prometheus_prefix = "kagome" + args = [ "--two", "--wasm-execution=Compiled", "-lparachain=debug", "-lruntime=debug", "-lreq_chunk_protocol=trace"] + +[[parachains]] +id = 2000 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=1" + + [parachains.collator] + name = "collator01" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=1", "--parachain-id=2000"] + +[[parachains]] +id = 2001 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=10" + + [parachains.collator] + name = "collator02" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2001", "--pvf-complexity=10"] + +[[parachains]] +id = 2002 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=100" + + [parachains.collator] + name = "collator03" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2002", "--pvf-complexity=100"] + +[[parachains]] +id = 2003 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=20000 --pvf-complexity=300" + + [parachains.collator] + name = "collator04" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=20000", "--parachain-id=2003", "--pvf-complexity=300"] + +[[parachains]] +id = 2004 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator05" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2004", "--pvf-complexity=300"] + +[[parachains]] +id = 2005 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=20000 --pvf-complexity=400" + + [parachains.collator] + name = "collator06" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=20000", "--pvf-complexity=400", "--parachain-id=2005"] + +[[parachains]] +id = 2006 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator07" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=300", "--parachain-id=2006"] + +[[parachains]] +id = 2007 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator08" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=300", "--parachain-id=2007"] + +[types.Header] +number = "u64" +parent_hash = "Hash" +post_state = "Hash" diff --git a/zombienet/polkadot/functional/0001-parachains-pvf-kagome-minority.zndsl b/zombienet/polkadot/functional/0001-parachains-pvf-kagome-minority.zndsl new file mode 100644 index 0000000000..fcda5dda5f --- /dev/null +++ b/zombienet/polkadot/functional/0001-parachains-pvf-kagome-minority.zndsl @@ -0,0 +1,84 @@ +Description: PVF preparation & execution time +Network: ./0001-parachains-pvf-kagome-minority.toml +Creds: config + +# Check authority status. +alice: reports node_roles is 4 +bob: reports node_roles is 4 +charlie: reports node_roles is 4 +dave: reports node_roles is 4 +eve: reports node_roles is 4 +ferdie: reports node_roles is 4 +one: reports node_roles is 4 +two: reports node_roles is 4 + +# Ensure parachains are registered. +alice: parachain 2000 is registered within 60 seconds +bob: parachain 2001 is registered within 60 seconds +charlie: parachain 2002 is registered within 60 seconds +dave: parachain 2003 is registered within 60 seconds +ferdie: parachain 2004 is registered within 60 seconds +eve: parachain 2005 is registered within 60 seconds +one: parachain 2006 is registered within 60 seconds +two: parachain 2007 is registered within 60 seconds + +# Ensure parachains made progress. +alice: parachain 2000 block height is at least 10 within 300 seconds +alice: parachain 2001 block height is at least 10 within 300 seconds +alice: parachain 2002 block height is at least 10 within 300 seconds +alice: parachain 2003 block height is at least 10 within 300 seconds +alice: parachain 2004 block height is at least 10 within 300 seconds +alice: parachain 2005 block height is at least 10 within 300 seconds +alice: parachain 2006 block height is at least 10 within 300 seconds +alice: parachain 2007 block height is at least 10 within 300 seconds + +alice: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds + +# Check preparation time is under 10s. +# Check all buckets <= 10. +alice: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds +bob: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds +charlie: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds +dave: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds +ferdie: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds +eve: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds +one: reports histogram kagome_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds +two: reports histogram kagome_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds + +# Check all buckets >= 20. +alice: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds +bob: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds +charlie: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds +dave: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds +ferdie: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds +eve: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds +one: reports histogram kagome_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds +two: reports histogram kagome_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds + +# Check execution time. +# There are two different timeout conditions: DEFAULT_BACKING_EXECUTION_TIMEOUT(2s) and +# DEFAULT_APPROVAL_EXECUTION_TIMEOUT(12s). Currently these are not differentiated by metrics +# because the metrics are defined in `polkadot-node-core-pvf` which is a level below +# the relevant subsystems. +# That being said, we will take the simplifying assumption of testing only the +# 2s timeout. +# We do this check by ensuring all executions fall into bucket le="2" or lower. +# First, check if we have at least 1 sample, but we should have many more. +alice: reports histogram polkadot_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds +bob: reports histogram polkadot_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds +charlie: reports histogram polkadot_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds +dave: reports histogram polkadot_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds +ferdie: reports histogram polkadot_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds +eve: reports histogram polkadot_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds +one: reports histogram kagome_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds +two: reports histogram kagome_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds + +# Check if we have no samples > 2s. +alice: reports histogram polkadot_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds +bob: reports histogram polkadot_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds +charlie: reports histogram polkadot_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds +dave: reports histogram polkadot_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds +ferdie: reports histogram polkadot_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds +eve: reports histogram polkadot_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds +one: reports histogram kagome_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds +two: reports histogram kagome_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds diff --git a/zombienet/polkadot/functional/0003-beefy-and-mmr-kagome-minority.toml b/zombienet/polkadot/functional/0003-beefy-and-mmr-kagome-minority.toml new file mode 100644 index 0000000000..2ab369de9d --- /dev/null +++ b/zombienet/polkadot/functional/0003-beefy-and-mmr-kagome-minority.toml @@ -0,0 +1,22 @@ +[settings] +timeout = 1000 + +[relaychain] +chain = "rococo-local" +command = "polkadot" + +[[relaychain.node_groups]] +name = "validator" +count = 3 +args = ["--log=beefy=debug", "--enable-offchain-indexing=true"] + +[[relaychain.node_groups]] +name = "kagome-validator" +command = "kagome" +prometheus_prefix = "kagome" +count = 1 +args = ["--log=beefy=debug", "--enable-offchain-indexing=true", "--wasm-execution=Compiled"] + +[[relaychain.nodes]] +name = "validator-unstable" +args = ["--log=beefy=debug", "--enable-offchain-indexing=true"] diff --git a/zombienet/polkadot/functional/0003-beefy-and-mmr-kagome-minority.zndsl b/zombienet/polkadot/functional/0003-beefy-and-mmr-kagome-minority.zndsl new file mode 100644 index 0000000000..dfcd603c8c --- /dev/null +++ b/zombienet/polkadot/functional/0003-beefy-and-mmr-kagome-minority.zndsl @@ -0,0 +1,42 @@ +Description: Test BEEFY voting and finality, test MMR proofs. Assumes Rococo sessions of 1 minute. +Network: ./0003-beefy-and-mmr-kagome-minority.toml +Creds: config + +# Check authority status. +validator: reports node_roles is 4 +validator-unstable: reports node_roles is 4 + +# BEEFY sanity checks. +validator: reports substrate_beefy_validator_set_id is 0 +validator-unstable: reports substrate_beefy_validator_set_id is 0 + +# Verify voting happens and 1st mandatory block is finalized within 1st session. +validator: reports substrate_beefy_best_block is at least 1 within 60 seconds +validator-unstable: reports substrate_beefy_best_block is at least 1 within 60 seconds + +# Pause validator-unstable and test chain is making progress without it. +validator-unstable: pause + +# Verify validator sets get changed on new sessions. +validator: reports substrate_beefy_validator_set_id is at least 1 within 70 seconds +kagome-validator: reports kagome_beefy_validator_set_id is at least 1 within 70 seconds +# Check next session too. +validator: reports substrate_beefy_validator_set_id is at least 2 within 130 seconds +kagome-validator: reports kagome_beefy_validator_set_id is at least 2 within 130 seconds + +# Verify voting happens and blocks are being finalized for new sessions too: +# since we verified we're at least in the 3rd session, verify BEEFY finalized mandatory #21. +validator: reports substrate_beefy_best_block is at least 21 within 130 seconds +kagome-validator: reports kagome_beefy_best_block is at least 21 within 130 seconds + +# Custom JS to test BEEFY RPCs. +validator-0: js-script ./0003-beefy-finalized-heads.js with "validator-0,validator-1,validator-2,kagome-validator-0" return is 1 within 5 seconds + +# Custom JS to test MMR RPCs. +validator: js-script ./0003-mmr-leaves.js with "21" return is 1 within 5 seconds +validator: js-script ./0003-mmr-generate-and-verify-proof.js with "validator-0,validator-1,validator-2,kagome-validator-0" return is 1 within 5 seconds + +# Resume validator-unstable and verify it imports all BEEFY justification and catches up. +validator-unstable: resume +validator-unstable: reports substrate_beefy_validator_set_id is at least 2 within 30 seconds +validator-unstable: reports substrate_beefy_best_block is at least 21 within 30 seconds