Add KAGOME minority tests for PVF preparation and BEEFY voting

.github/workflows/zombie-tests.yaml

@@ -249,7 +249,9 @@ jobs:
         platform_config: ${{ fromJSON(needs.setup_matrix.outputs.platforms) }}
         scenario: [
           { name: "0001 PVF preparation & execution time", test: "test-polkadot-functional-0001-parachains-pvf" },
+          { name: "0001 PVF preparation & execution time with KAGOME minority", test: "test-polkadot-functional-0001-parachains-pvf-kagome-minority" },
           { name: "0003 BEEFY voting and finality, test MMR proofs. Assumes Rococo sessions of 1 minute", test: "test-polkadot-functional-0003-beefy-and-mmr" },
+          { name: "0003 BEEFY voting and finality, test MMR proofs. Assumes Rococo sessions of 1 minute with KAGOME minority", test: "test-polkadot-functional-0003-beefy-and-mmr-kagome-minority" },
           { name: "0007 Test if disputes triggered on finalized blocks within scope always end as valid", test: "test-polkadot-functional-0007-dispute-freshly-finalized" },
           { name: "0008 Test if disputes triggered on finalized blocks out of scope never get to be confirmed and concluded", test: "test-polkadot-functional-0008-dispute-old-finalized" },
           { name: "0009 Approval voting coalescing does not lag finality", test: "test-polkadot-functional-0009-approval-voting-coalescing" },

zombienet/docker/Makefile.include/zombietests.mk

@@ -1,11 +1,16 @@
 test-polkadot-functional-0001-parachains-pvf:
 	$(call run_test, "kagome/zombienet/polkadot/functional/0001-parachains-pvf.zndsl")
 
+test-polkadot-functional-0001-parachains-pvf-kagome-minority:
+	$(call run_test, "kagome/zombienet/polkadot/functional/0001-parachains-pvf-kagome-minority.zndsl")
+
 test-polkadot-functional-0002-parachains-disputes:
 	$(call run_test, "kagome/zombienet/polkadot/functional/0002-parachains-disputes.zndsl")
 
 test-polkadot-functional-0003-beefy-and-mmr:
 	$(call run_test, "kagome/zombienet/polkadot/functional/0003-beefy-and-mmr.zndsl")
+test-polkadot-functional-0003-beefy-and-mmr-kagome-minority:
+	$(call run_test, "kagome/zombienet/polkadot/functional/0003-beefy-and-mmr-kagome-minority.zndsl")
 
 test-polkadot-functional-0004-parachains-garbage-candidate:
 	$(call run_test, "kagome/zombienet/polkadot/functional/0004-parachains-garbage-candidate.zndsl")

zombienet/polkadot/functional/0001-parachains-pvf-kagome-minority.toml

@@ -0,0 +1,130 @@
+[settings]
+timeout = 1000
+
+[relaychain]
+chain = "rococo-local"
+
+[relaychain.default_resources]
+limits = { memory = "4G", cpu = "2" }
+requests = { memory = "2G", cpu = "1" }
+
+  [[relaychain.nodes]]
+  name = "alice"
+  args = [ "--alice", "-lparachain=debug,runtime=debug" ]
+
+  [[relaychain.nodes]]
+  name = "bob"
+  args = [ "--bob", "-lparachain=debug,runtime=debug" ]
+
+  [[relaychain.nodes]]
+  name = "charlie"
+  args = [ "--charlie", "-lparachain=debug,runtime=debug" ]
+
+  [[relaychain.nodes]]
+  name = "dave"
+  args = [ "--dave", "-lparachain=debug,runtime=debug" ]
+
+  [[relaychain.nodes]]
+  name = "ferdie"
+  args = [ "--ferdie", "-lparachain=debug,runtime=debug"]
+
+  [[relaychain.nodes]]
+  name = "eve"
+  args = [ "--eve", "-lparachain=debug,runtime=debug"]
+
+  [[relaychain.nodes]]
+  name = "one"
+  command = "kagome"
+  prometheus_prefix = "kagome"
+  args = [ "--one", "--wasm-execution=Compiled", "-lparachain=debug", "-lnetwork=debug"]
+
+  [[relaychain.nodes]]
+  name = "two"
+  command = "kagome"
+  prometheus_prefix = "kagome"
+  args = [ "--two", "--wasm-execution=Compiled", "-lparachain=debug", "-lruntime=debug", "-lreq_chunk_protocol=trace"]
+
+[[parachains]]
+id = 2000
+addToGenesis = true
+genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=1"
+
+  [parachains.collator]
+  name = "collator01"
+  command = "undying-collator"
+  args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=1", "--parachain-id=2000"]
+
+[[parachains]]
+id = 2001
+addToGenesis = true
+genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=10"
+
+  [parachains.collator]
+  name = "collator02"
+  command = "undying-collator"
+  args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2001", "--pvf-complexity=10"]
+
+[[parachains]]
+id = 2002
+addToGenesis = true
+genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=100"
+
+  [parachains.collator]
+  name = "collator03"
+  command = "undying-collator"
+  args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2002", "--pvf-complexity=100"]
+
+[[parachains]]
+id = 2003
+addToGenesis = true
+genesis_state_generator = "undying-collator export-genesis-state --pov-size=20000 --pvf-complexity=300"
+
+  [parachains.collator]
+  name = "collator04"
+  command = "undying-collator"
+  args = ["-lparachain=debug", "--pov-size=20000", "--parachain-id=2003", "--pvf-complexity=300"]
+
+[[parachains]]
+id = 2004
+addToGenesis = true
+genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300"
+
+  [parachains.collator]
+  name = "collator05"
+  command = "undying-collator"
+  args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2004", "--pvf-complexity=300"]
+
+[[parachains]]
+id = 2005
+addToGenesis = true
+genesis_state_generator = "undying-collator export-genesis-state --pov-size=20000 --pvf-complexity=400"
+
+  [parachains.collator]
+  name = "collator06"
+  command = "undying-collator"
+  args = ["-lparachain=debug", "--pov-size=20000",  "--pvf-complexity=400", "--parachain-id=2005"]
+
+[[parachains]]
+id = 2006
+addToGenesis = true
+genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300"
+
+  [parachains.collator]
+  name = "collator07"
+  command = "undying-collator"
+  args = ["-lparachain=debug", "--pov-size=100000",  "--pvf-complexity=300", "--parachain-id=2006"]
+
+[[parachains]]
+id = 2007
+addToGenesis = true
+genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300"
+
+  [parachains.collator]
+  name = "collator08"
+  command = "undying-collator"
+  args = ["-lparachain=debug", "--pov-size=100000",  "--pvf-complexity=300", "--parachain-id=2007"]
+
+[types.Header]
+number = "u64"
+parent_hash = "Hash"
+post_state = "Hash"

zombienet/polkadot/functional/0001-parachains-pvf-kagome-minority.zndsl

@@ -0,0 +1,84 @@
+Description: PVF preparation & execution time
+Network: ./0001-parachains-pvf-kagome-minority.toml
+Creds: config
+
+# Check authority status.
+alice: reports node_roles is 4
+bob: reports node_roles is 4
+charlie: reports node_roles is 4
+dave: reports node_roles is 4
+eve: reports node_roles is 4
+ferdie: reports node_roles is 4
+one: reports node_roles is 4
+two: reports node_roles is 4
+
+# Ensure parachains are registered.
+alice: parachain 2000 is registered within 60 seconds
+bob: parachain 2001 is registered within 60 seconds
+charlie: parachain 2002 is registered within 60 seconds
+dave: parachain 2003 is registered within 60 seconds
+ferdie: parachain 2004 is registered within 60 seconds
+eve: parachain 2005 is registered within 60 seconds
+one: parachain 2006 is registered within 60 seconds
+two: parachain 2007 is registered within 60 seconds
+
+# Ensure parachains made progress.
+alice: parachain 2000 block height is at least 10 within 300 seconds
+alice: parachain 2001 block height is at least 10 within 300 seconds
+alice: parachain 2002 block height is at least 10 within 300 seconds
+alice: parachain 2003 block height is at least 10 within 300 seconds
+alice: parachain 2004 block height is at least 10 within 300 seconds
+alice: parachain 2005 block height is at least 10 within 300 seconds
+alice: parachain 2006 block height is at least 10 within 300 seconds
+alice: parachain 2007 block height is at least 10 within 300 seconds
+
+alice: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds
+
+# Check preparation time is under 10s.
+# Check all buckets <= 10.
+alice: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds
+bob: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds
+charlie: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds
+dave: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds
+ferdie: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds
+eve: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds
+one: reports histogram kagome_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds
+two: reports histogram kagome_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds
+
+# Check all buckets >= 20.
+alice: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds
+bob: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds
+charlie: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds
+dave: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds
+ferdie: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds
+eve: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds
+one: reports histogram kagome_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds
+two: reports histogram kagome_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds
+
+# Check execution time.
+# There are two different timeout conditions: DEFAULT_BACKING_EXECUTION_TIMEOUT(2s) and
+# DEFAULT_APPROVAL_EXECUTION_TIMEOUT(12s). Currently these are not differentiated by metrics
+# because the metrics are defined in `polkadot-node-core-pvf` which is a level below
+# the relevant subsystems.
+# That being said, we will take the simplifying assumption of testing only the
+# 2s timeout.
+# We do this check by ensuring all executions fall into bucket le="2" or lower.
+# First, check if we have at least 1 sample, but we should have many more.
+alice: reports histogram polkadot_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds
+bob: reports histogram polkadot_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds
+charlie: reports histogram polkadot_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds
+dave: reports histogram polkadot_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds
+ferdie: reports histogram polkadot_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds
+eve: reports histogram polkadot_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds
+one: reports histogram kagome_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds
+two: reports histogram kagome_pvf_execution_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2"] within 10 seconds
+
+# Check if we have no samples > 2s.
+alice: reports histogram polkadot_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds
+bob: reports histogram polkadot_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds
+charlie: reports histogram polkadot_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds
+dave: reports histogram polkadot_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds
+ferdie: reports histogram polkadot_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds
+eve: reports histogram polkadot_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds
+one: reports histogram kagome_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds
+two: reports histogram kagome_pvf_execution_time has 0 samples in buckets ["3", "4", "5", "6", "+Inf"] within 10 seconds

zombienet/polkadot/functional/0003-beefy-and-mmr-kagome-minority.toml

@@ -0,0 +1,22 @@
+[settings]
+timeout = 1000
+
+[relaychain]
+chain = "rococo-local"
+command = "polkadot"
+
+[[relaychain.node_groups]]
+name = "validator"
+count = 3
+args = ["--log=beefy=debug", "--enable-offchain-indexing=true"]
+
+[[relaychain.node_groups]]
+name = "kagome-validator"
+command = "kagome"
+prometheus_prefix = "kagome"
+count = 1
+args = ["--log=beefy=debug", "--enable-offchain-indexing=true", "--wasm-execution=Compiled"]
+
+[[relaychain.nodes]]
+name = "validator-unstable"
+args = ["--log=beefy=debug", "--enable-offchain-indexing=true"]

zombienet/polkadot/functional/0003-beefy-and-mmr-kagome-minority.zndsl

@@ -0,0 +1,42 @@
+Description: Test BEEFY voting and finality, test MMR proofs. Assumes Rococo sessions of 1 minute.
+Network: ./0003-beefy-and-mmr-kagome-minority.toml
+Creds: config
+
+# Check authority status.
+validator: reports node_roles is 4
+validator-unstable: reports node_roles is 4
+
+# BEEFY sanity checks.
+validator: reports substrate_beefy_validator_set_id is 0
+validator-unstable: reports substrate_beefy_validator_set_id is 0
+
+# Verify voting happens and 1st mandatory block is finalized within 1st session.
+validator: reports substrate_beefy_best_block is at least 1 within 60 seconds
+validator-unstable: reports substrate_beefy_best_block is at least 1 within 60 seconds
+
+# Pause validator-unstable and test chain is making progress without it.
+validator-unstable: pause
+
+# Verify validator sets get changed on new sessions.
+validator: reports substrate_beefy_validator_set_id is at least 1 within 70 seconds
+kagome-validator: reports kagome_beefy_validator_set_id is at least 1 within 70 seconds
+# Check next session too.
+validator: reports substrate_beefy_validator_set_id is at least 2 within 130 seconds
+kagome-validator: reports kagome_beefy_validator_set_id is at least 2 within 130 seconds
+
+# Verify voting happens and blocks are being finalized for new sessions too:
+# since we verified we're at least in the 3rd session, verify BEEFY finalized mandatory #21.
+validator: reports substrate_beefy_best_block is at least 21 within 130 seconds
+kagome-validator: reports kagome_beefy_best_block is at least 21 within 130 seconds
+
+# Custom JS to test BEEFY RPCs.
+validator-0: js-script ./0003-beefy-finalized-heads.js with "validator-0,validator-1,validator-2,kagome-validator-0" return is 1 within 5 seconds
+
+# Custom JS to test MMR RPCs.
+validator: js-script ./0003-mmr-leaves.js with "21" return is 1 within 5 seconds
+validator: js-script ./0003-mmr-generate-and-verify-proof.js with "validator-0,validator-1,validator-2,kagome-validator-0" return is 1 within 5 seconds
+
+# Resume validator-unstable and verify it imports all BEEFY justification and catches up.
+validator-unstable: resume
+validator-unstable: reports substrate_beefy_validator_set_id is at least 2 within 30 seconds
+validator-unstable: reports substrate_beefy_best_block is at least 21 within 30 seconds