From 2ebe3eef0305eed9db0a786d33bc511d17e915f3 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Tue, 17 Nov 2020 10:35:50 +0700 Subject: [PATCH 1/5] Improve initiateValidatorExit by processing in batch --- .../src/fast/block/initiateValidatorExit.ts | 39 +++++++++++++++++++ .../src/fast/block/processOperations.ts | 9 +++-- .../src/fast/block/processVoluntaryExit.ts | 22 ++++++++++- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts b/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts index 6d0c2bba8c1f..da2db6d5f9ff 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts @@ -5,6 +5,9 @@ import {FAR_FUTURE_EPOCH} from "../../constants"; import {computeActivationExitEpoch, getChurnLimit} from "../../util"; import {EpochContext} from "../util"; +/** + * Old implementation following the spec, mainly for the spec test. + */ export function initiateValidatorExit(epochCtx: EpochContext, state: BeaconState, index: ValidatorIndex): void { const config = epochCtx.config; // return if validator already initiated exit @@ -29,3 +32,39 @@ export function initiateValidatorExit(epochCtx: EpochContext, state: BeaconState validator.exitEpoch = exitQueueEpoch; validator.withdrawableEpoch = validator.exitEpoch + config.params.MIN_VALIDATOR_WITHDRAWABILITY_DELAY; } + +/** + * Optimized version of initiateValidatorExit where we process validators in batch. + * The main thing is to loop through state.validators exactly 1 time. + */ +export function initiateMultipleValidatorExits( + epochCtx: EpochContext, + state: BeaconState, + indexes: ValidatorIndex[] +): void { + const config = epochCtx.config; + // compute exit queue epoch + const validatorExitEpochs = readOnlyMap(state.validators, (v) => v.exitEpoch); + const exitEpochs = validatorExitEpochs.filter((exitEpoch) => exitEpoch !== FAR_FUTURE_EPOCH); + const churnLimit = getChurnLimit(config, epochCtx.currentShuffling.activeIndices.length); + const currentEpoch = epochCtx.currentShuffling.epoch; + const activationExitEpoch = computeActivationExitEpoch(config, currentEpoch); + for (const index of indexes) { + // return if validator already initiated exit + const validator = state.validators[index]; + if (validator.exitEpoch !== FAR_FUTURE_EPOCH) { + return; + } + let exitQueueEpoch = Math.max(...exitEpochs, activationExitEpoch); + const exitQueueChurn = validatorExitEpochs.filter((exitEpoch) => exitEpoch === exitQueueEpoch).length; + if (exitQueueChurn >= churnLimit) { + exitQueueEpoch += 1; + } + + // set validator exit epoch and withdrawable epoch + validator.exitEpoch = exitQueueEpoch; + validator.withdrawableEpoch = exitQueueEpoch + config.params.MIN_VALIDATOR_WITHDRAWABILITY_DELAY; + validatorExitEpochs[index] = exitQueueEpoch; + exitEpochs.push(exitQueueEpoch); + } +} diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/processOperations.ts b/packages/lodestar-beacon-state-transition/src/fast/block/processOperations.ts index 0259ac8b6678..612d55207567 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/processOperations.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/processOperations.ts @@ -1,4 +1,4 @@ -import {List, readOnlyForEach} from "@chainsafe/ssz"; +import {List, readOnlyForEach, readOnlyMap} from "@chainsafe/ssz"; import { Attestation, AttesterSlashing, @@ -14,7 +14,8 @@ import {processProposerSlashing} from "./processProposerSlashing"; import {processAttesterSlashing} from "./processAttesterSlashing"; import {processAttestation} from "./processAttestation"; import {processDeposit} from "./processDeposit"; -import {processVoluntaryExit} from "./processVoluntaryExit"; +import {verifyVoluntaryExit} from "./processVoluntaryExit"; +import {initiateMultipleValidatorExits} from "./initiateValidatorExit"; type Operation = ProposerSlashing | AttesterSlashing | Attestation | Deposit | VoluntaryExit; type OperationFunction = (epochCtx: EpochContext, state: BeaconState, op: Operation, verify: boolean) => void; @@ -41,10 +42,12 @@ export function processOperations( [body.attesterSlashings, processAttesterSlashing], [body.attestations, processAttestation], [body.deposits, processDeposit], - [body.voluntaryExits, processVoluntaryExit], + [body.voluntaryExits, verifyVoluntaryExit], ] as [List, OperationFunction][]).forEach(([operations, processOp]) => { readOnlyForEach(operations, (op) => { processOp(epochCtx, state, op, verifySignatures); }); }); + const indexes = readOnlyMap(body.voluntaryExits, (voluntaryExit) => voluntaryExit.message.validatorIndex); + initiateMultipleValidatorExits(epochCtx, state, indexes); } diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/processVoluntaryExit.ts b/packages/lodestar-beacon-state-transition/src/fast/block/processVoluntaryExit.ts index cfd355bdf20b..a9dba320a3d2 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/processVoluntaryExit.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/processVoluntaryExit.ts @@ -6,11 +6,31 @@ import {computeSigningRoot, getDomain, isActiveValidator} from "../../util"; import {EpochContext} from "../util"; import {initiateValidatorExit} from "./initiateValidatorExit"; +/** + * This is for the spec test only. + * Use verifyVoluntaryExit to process validator exits in batch. + */ export function processVoluntaryExit( epochCtx: EpochContext, state: BeaconState, signedVoluntaryExit: SignedVoluntaryExit, verifySignature = true +): void { + verifyVoluntaryExit(epochCtx, state, signedVoluntaryExit, verifySignature); + // initiate exit + const voluntaryExit = signedVoluntaryExit.message; + initiateValidatorExit(epochCtx, state, voluntaryExit.validatorIndex); +} + +/** + * The same to processVoluntaryExit without calling initiateValidatorExit. + * We'll do it in batch. + */ +export function verifyVoluntaryExit( + epochCtx: EpochContext, + state: BeaconState, + signedVoluntaryExit: SignedVoluntaryExit, + verifySignature = true ): void { const config = epochCtx.config; const voluntaryExit = signedVoluntaryExit.message; @@ -48,6 +68,4 @@ export function processVoluntaryExit( throw new Error("VoluntaryExit has an invalid signature"); } } - // initiate exit - initiateValidatorExit(epochCtx, state, voluntaryExit.validatorIndex); } From 34dcb15391e9dfb575543dee1e2f1acb79b715b1 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Wed, 18 Nov 2020 05:46:51 +0700 Subject: [PATCH 2/5] initiateMultipleValidatorExits: handle empty validator index array --- .../src/fast/block/initiateValidatorExit.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts b/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts index da2db6d5f9ff..93a1de5edfd1 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts @@ -40,8 +40,9 @@ export function initiateValidatorExit(epochCtx: EpochContext, state: BeaconState export function initiateMultipleValidatorExits( epochCtx: EpochContext, state: BeaconState, - indexes: ValidatorIndex[] + indexes: ValidatorIndex[] = [] ): void { + if (!indexes.length) return; const config = epochCtx.config; // compute exit queue epoch const validatorExitEpochs = readOnlyMap(state.validators, (v) => v.exitEpoch); From 8a253bb9f0c598d0d3185d2857ae0ad009193ffb Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Wed, 18 Nov 2020 09:09:58 +0700 Subject: [PATCH 3/5] Change the logic of return to continue inside for loop --- .../src/fast/block/initiateValidatorExit.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts b/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts index 93a1de5edfd1..d33272a868e4 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts @@ -42,7 +42,7 @@ export function initiateMultipleValidatorExits( state: BeaconState, indexes: ValidatorIndex[] = [] ): void { - if (!indexes.length) return; + if (!indexes || indexes.length === 0) return; const config = epochCtx.config; // compute exit queue epoch const validatorExitEpochs = readOnlyMap(state.validators, (v) => v.exitEpoch); @@ -51,10 +51,10 @@ export function initiateMultipleValidatorExits( const currentEpoch = epochCtx.currentShuffling.epoch; const activationExitEpoch = computeActivationExitEpoch(config, currentEpoch); for (const index of indexes) { - // return if validator already initiated exit + // continue if validator already initiated exit const validator = state.validators[index]; if (validator.exitEpoch !== FAR_FUTURE_EPOCH) { - return; + continue; } let exitQueueEpoch = Math.max(...exitEpochs, activationExitEpoch); const exitQueueChurn = validatorExitEpochs.filter((exitEpoch) => exitEpoch === exitQueueEpoch).length; From c8f061d93f824d0c3f8f214b0572a31361cf54ea Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Thu, 19 Nov 2020 14:00:28 +0700 Subject: [PATCH 4/5] Address comments and get through spec tests --- .../src/fast/block/index.ts | 4 +-- .../src/fast/block/initiateValidatorExit.ts | 16 +++++++---- .../src/fast/block/processOperations.ts | 10 +++---- .../src/fast/block/processVoluntaryExit.ts | 28 +++++++++++++++---- .../voluntaryExit/voluntary_exit_fast.test.ts | 7 +++-- 5 files changed, 43 insertions(+), 22 deletions(-) diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/index.ts b/packages/lodestar-beacon-state-transition/src/fast/block/index.ts index 36b607fd5145..6bbf2bf1be31 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/index.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/index.ts @@ -9,7 +9,7 @@ import {processAttestation} from "./processAttestation"; import {processAttesterSlashing} from "./processAttesterSlashing"; import {processDeposit} from "./processDeposit"; import {processProposerSlashing} from "./processProposerSlashing"; -import {processVoluntaryExit} from "./processVoluntaryExit"; +import {processVoluntaryExits} from "./processVoluntaryExit"; export { processBlockHeader, @@ -20,7 +20,7 @@ export { processAttesterSlashing, processDeposit, processProposerSlashing, - processVoluntaryExit, + processVoluntaryExits, }; export function processBlock( diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts b/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts index d33272a868e4..02a333e55a58 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts @@ -1,12 +1,12 @@ -import {readOnlyMap} from "@chainsafe/ssz"; -import {BeaconState, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {readOnlyForEach, readOnlyMap} from "@chainsafe/ssz"; +import {BeaconState, Validator, ValidatorIndex} from "@chainsafe/lodestar-types"; import {FAR_FUTURE_EPOCH} from "../../constants"; import {computeActivationExitEpoch, getChurnLimit} from "../../util"; import {EpochContext} from "../util"; /** - * Old implementation following the spec, mainly for the spec test. + * Initiate a single validator exit. */ export function initiateValidatorExit(epochCtx: EpochContext, state: BeaconState, index: ValidatorIndex): void { const config = epochCtx.config; @@ -45,8 +45,14 @@ export function initiateMultipleValidatorExits( if (!indexes || indexes.length === 0) return; const config = epochCtx.config; // compute exit queue epoch - const validatorExitEpochs = readOnlyMap(state.validators, (v) => v.exitEpoch); - const exitEpochs = validatorExitEpochs.filter((exitEpoch) => exitEpoch !== FAR_FUTURE_EPOCH); + const validatorExitEpochs = []; + const exitEpochs = []; + readOnlyForEach(state.validators, (v: Validator) => { + validatorExitEpochs.push(v.exitEpoch); + if (v.exitEpoch !== FAR_FUTURE_EPOCH) { + exitEpochs.push(v.exitEpoch); + } + }); const churnLimit = getChurnLimit(config, epochCtx.currentShuffling.activeIndices.length); const currentEpoch = epochCtx.currentShuffling.epoch; const activationExitEpoch = computeActivationExitEpoch(config, currentEpoch); diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/processOperations.ts b/packages/lodestar-beacon-state-transition/src/fast/block/processOperations.ts index 612d55207567..36ad8ad283bc 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/processOperations.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/processOperations.ts @@ -1,4 +1,4 @@ -import {List, readOnlyForEach, readOnlyMap} from "@chainsafe/ssz"; +import {List, readOnlyForEach} from "@chainsafe/ssz"; import { Attestation, AttesterSlashing, @@ -14,8 +14,7 @@ import {processProposerSlashing} from "./processProposerSlashing"; import {processAttesterSlashing} from "./processAttesterSlashing"; import {processAttestation} from "./processAttestation"; import {processDeposit} from "./processDeposit"; -import {verifyVoluntaryExit} from "./processVoluntaryExit"; -import {initiateMultipleValidatorExits} from "./initiateValidatorExit"; +import {processVoluntaryExits} from "./processVoluntaryExit"; type Operation = ProposerSlashing | AttesterSlashing | Attestation | Deposit | VoluntaryExit; type OperationFunction = (epochCtx: EpochContext, state: BeaconState, op: Operation, verify: boolean) => void; @@ -42,12 +41,11 @@ export function processOperations( [body.attesterSlashings, processAttesterSlashing], [body.attestations, processAttestation], [body.deposits, processDeposit], - [body.voluntaryExits, verifyVoluntaryExit], ] as [List, OperationFunction][]).forEach(([operations, processOp]) => { readOnlyForEach(operations, (op) => { processOp(epochCtx, state, op, verifySignatures); }); }); - const indexes = readOnlyMap(body.voluntaryExits, (voluntaryExit) => voluntaryExit.message.validatorIndex); - initiateMultipleValidatorExits(epochCtx, state, indexes); + // process voluntary exits in batch to optimize performance + processVoluntaryExits(epochCtx, state, body.voluntaryExits, verifySignatures); } diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/processVoluntaryExit.ts b/packages/lodestar-beacon-state-transition/src/fast/block/processVoluntaryExit.ts index a9dba320a3d2..03c8093205f9 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/processVoluntaryExit.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/processVoluntaryExit.ts @@ -1,14 +1,14 @@ import {Signature} from "@chainsafe/bls"; -import {BeaconState, SignedVoluntaryExit} from "@chainsafe/lodestar-types"; +import {BeaconState, SignedVoluntaryExit, ValidatorIndex} from "@chainsafe/lodestar-types"; +import {List, readOnlyForEach} from "@chainsafe/ssz"; import {DomainType, FAR_FUTURE_EPOCH} from "../../constants"; import {computeSigningRoot, getDomain, isActiveValidator} from "../../util"; import {EpochContext} from "../util"; -import {initiateValidatorExit} from "./initiateValidatorExit"; +import {initiateMultipleValidatorExits, initiateValidatorExit} from "./initiateValidatorExit"; /** - * This is for the spec test only. - * Use verifyVoluntaryExit to process validator exits in batch. + * Process a single voluntary exit, for backward reference only. */ export function processVoluntaryExit( epochCtx: EpochContext, @@ -22,11 +22,27 @@ export function processVoluntaryExit( initiateValidatorExit(epochCtx, state, voluntaryExit.validatorIndex); } +/** + * Process voluntary exits in batch + */ +export function processVoluntaryExits( + epochCtx: EpochContext, + state: BeaconState, + signedVoluntaryExits: List, + verifySignature = true +): void { + const indexes = new Set(); + readOnlyForEach(signedVoluntaryExits, (signedVoluntaryExit) => { + verifyVoluntaryExit(epochCtx, state, signedVoluntaryExit, verifySignature); + indexes.add(signedVoluntaryExit.message.validatorIndex); + }); + initiateMultipleValidatorExits(epochCtx, state, Array.from(indexes)); +} + /** * The same to processVoluntaryExit without calling initiateValidatorExit. - * We'll do it in batch. */ -export function verifyVoluntaryExit( +function verifyVoluntaryExit( epochCtx: EpochContext, state: BeaconState, signedVoluntaryExit: SignedVoluntaryExit, diff --git a/packages/spec-test-runner/test/spec/operations/voluntaryExit/voluntary_exit_fast.test.ts b/packages/spec-test-runner/test/spec/operations/voluntaryExit/voluntary_exit_fast.test.ts index d46b4fd94d56..0a19a9ede86f 100644 --- a/packages/spec-test-runner/test/spec/operations/voluntaryExit/voluntary_exit_fast.test.ts +++ b/packages/spec-test-runner/test/spec/operations/voluntaryExit/voluntary_exit_fast.test.ts @@ -1,12 +1,13 @@ import {join} from "path"; import {expect} from "chai"; -import {BeaconState} from "@chainsafe/lodestar-types"; +import {BeaconState, SignedVoluntaryExit} from "@chainsafe/lodestar-types"; import {config} from "@chainsafe/lodestar-config/lib/presets/mainnet"; import {EpochContext} from "@chainsafe/lodestar-beacon-state-transition"; -import {processVoluntaryExit} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/block"; +import {processVoluntaryExits} from "@chainsafe/lodestar-beacon-state-transition/lib/fast/block"; import {describeDirectorySpecTest} from "@chainsafe/lodestar-spec-test-util/lib/single"; import {IProcessVoluntaryExitTestCase} from "./type"; import {SPEC_TEST_LOCATION} from "../../../utils/specTestCases"; +import {List} from "@chainsafe/ssz"; describeDirectorySpecTest( "process voluntary exit mainnet", @@ -15,7 +16,7 @@ describeDirectorySpecTest( const state = testcase.pre; const epochCtx = new EpochContext(config); epochCtx.loadState(state); - processVoluntaryExit(epochCtx, state, testcase.voluntary_exit); + processVoluntaryExits(epochCtx, state, [testcase.voluntary_exit] as List); return state; }, { From 3961799e7a8fe4bc42afe9631d76a1c87fcb9fdb Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Fri, 20 Nov 2020 05:50:52 +0700 Subject: [PATCH 5/5] Use initiateMultipleValidatorExits for slashValidator() --- .../src/fast/block/initiateValidatorExit.ts | 2 +- .../src/fast/block/slashValidator.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts b/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts index 02a333e55a58..3eebf05e7f8f 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/initiateValidatorExit.ts @@ -6,7 +6,7 @@ import {computeActivationExitEpoch, getChurnLimit} from "../../util"; import {EpochContext} from "../util"; /** - * Initiate a single validator exit. + * Initiate a single validator exit, similar to the spec, for backward reference only. */ export function initiateValidatorExit(epochCtx: EpochContext, state: BeaconState, index: ValidatorIndex): void { const config = epochCtx.config; diff --git a/packages/lodestar-beacon-state-transition/src/fast/block/slashValidator.ts b/packages/lodestar-beacon-state-transition/src/fast/block/slashValidator.ts index 42964e7a289a..78c6e48e07ca 100644 --- a/packages/lodestar-beacon-state-transition/src/fast/block/slashValidator.ts +++ b/packages/lodestar-beacon-state-transition/src/fast/block/slashValidator.ts @@ -2,7 +2,7 @@ import {BeaconState, ValidatorIndex} from "@chainsafe/lodestar-types"; import {decreaseBalance, increaseBalance} from "../../util"; import {EpochContext} from "../util"; -import {initiateValidatorExit} from "./initiateValidatorExit"; +import {initiateMultipleValidatorExits} from "./initiateValidatorExit"; export function slashValidator( epochCtx: EpochContext, @@ -17,7 +17,7 @@ export function slashValidator( PROPOSER_REWARD_QUOTIENT, } = epochCtx.config.params; const epoch = epochCtx.currentShuffling.epoch; - initiateValidatorExit(epochCtx, state, slashedIndex); + initiateMultipleValidatorExits(epochCtx, state, [slashedIndex]); const validator = state.validators[slashedIndex]; validator.slashed = true; validator.withdrawableEpoch = Math.max(validator.withdrawableEpoch, epoch + EPOCHS_PER_SLASHINGS_VECTOR);