Improved Quorum-Genesis Docs

.gitignore

@@ -1,3 +1,4 @@
 .DS_Store
 node_modules/
-quorum-config.json
+quorum-config.json
+quorum-genesis.json

README.md

@@ -22,18 +22,81 @@ The `quorum-config.json` should be in the following format:
 
 ```json
 {
-  "threshold": 3,
+  "chainID": 1,
+  "makers": ["0xca843569e3427144cead5e4d5999a3d0ccf92b8e"],
   "voters": [
     "0x0fbdc686b912d7722dc86510934589e0aaf3b55a",
     "0x9186eb3d20cbd1f5f992a950d808c4495153abd5",
     "0x0638e1574728b6d862dd5d3a3e0942c3be47d996"
-  ],
-  "makers": ["0xca843569e3427144cead5e4d5999a3d0ccf92b8e"]
+  ]
 }
 ```
 
 Where:
+* `chainID` is mandatory & will be the block's `chainID`
+* `makers` is mandatory & will be used to set the blockmakers on the BlockVoting contract.  The number of makers is also used to calculate the initial difficulty (`EXPECTED_HASHRATE * numMakers * DESIRED_SECONDS_PER_BLOCK`).
+* `voters` is optional. They will be set as voters on the BlockMaking contract, but in the POW model, voters are unnecessary.  If specified, they will be included among the owners of the WeylGov contract.
+* `gasLimit` is optional, and if specified, will be included as the block's `gasLimit`.
 
-* `threshold` is the number of required voters
-* `voters` is a list of voter addresses
-* `makers` is a list of blockmaker addresses
+How Does EVM Storage Work
+---
+
+Fundamentally, this script uses a template and the above config to lay out initial storage values for our two governance smart contracts.  Its functions make more sense if you understand how Ethereum represents contract storage.  The genesis block has an `alloc` key which allows us to give any address some `code`, `storage`, or a `balance`.  `code` is for contract bytecode, `balance` is initial gas.  `storage` allows you to create initial values for the state variables declared in the smart contract's source.
+
+The genesis block represents state in a flat object.  Each state variable is referenced in the order it was declared in the source code; take a look at the sample contract below:
+
+```sol
+contract WeylExample {
+
+  uint public numOwners;
+  mapping(address => bool) private owners;
+
+  // Rest of contract...
+}
+```
+
+The first state variable is a uint, or more importantly, a simple [Value Type](https://solidity.readthedocs.io/en/v0.4.24/types.html#value-types).  Its key in the storage object will be `0` -- state variables are zero-indexed -- but converted into a padded hex string: `0x0...0`.  Its uint value is stored as an un-padded, `0x`-prefixed hex string, so with one owner, the Weyl contract's storage shape starts out like this: 
+
+```
+{
+  "alloc": {
+    "0x0...02A" : {
+      "code": "0x...",
+      "storage": {
+        "0x0000000000000000000000000000000000000000000000000000000000000000": "0x1"
+      }
+    }
+  }
+}
+```
+
+The second state variable is a [Mapping](https://solidity.readthedocs.io/en/v0.4.24/types.html#mappings).  The storage object is flat, so each value stored in the map gets its own key.  This key is calculated with the following steps:
+
+1. Pad the address which will be used to key into the map and the index of the variable (for our `owners` variable, `1`).
+2. Make a hex Buffer which is the address followed by the index.
+3. Return the `sha3` hash of this Buffer as a `0x`-prefixed hex string.
+
+Written in pseudocode, setting one address key in a map to `true` looks something like:
+```
+storage[ '0x' + sha3( pad(addressKey) + pad(mapIndex) )] = '0x1';
+```
+Where `addressKey` is the key on the map and `mapIndex` is that map's index in the list of state variables as declared in the sourcecode (in the sample case, `mapIndex = 1`).  Put this all together, and in the one-owner case, you end up with the following `storage` object:
+
+```
+{
+  "storage": {
+    "0x0000000000000000000000000000000000000000000000000000000000000000": "0x1",
+    "0xe9b6e9843417f0ea3a8ed4ff9903fd0b05391abe8cff9eae43d392f2221aefe9": "0x1"
+  }
+}
+```
+Similar procedures are used for initializing arrays, except with array indices instead of map keys.  
+
+Make sure to do extensive tests with any new genesis block, as misformatted memory will make functions return garbage values.  This script is only responsible for setting the variables shown above.  If we wanted to import governance history into a new genesis block, we would also need to set initial values for the other assorted state variables.
+
+So We Want to Hardfork...
+---
+
+The syntax described above was all we needed to know when we were initializing the BlockVoting & EximGov contracts.  The mappings we needed to set only stored booleans, but we have some mappings which store structs.  If we wanted to do a hard fork where we imported the full current state of the governance contract, we'd need to do research on how those are represented.
+
+One strategy to rebuild `storage` would be manually querying all of the locations where there are values.  `web3` provides a function to fetch the value at any specific index.  In practice, this means we would need to calculate all `map` and `array` indices ourselves, potentially using the transaction history, and then fetch the value stored in each one.  It's technically possible, but a significant amount of legwork on our end.  Definitely worth thinking through whether we really need to import *all* of this contract's historical state.

index.js

@@ -19,24 +19,52 @@ const REMAINDER_ADDR = '0x9153A2a04cc57B486AB82bC0bE341DCa367B7934';
 // Used to calculate the initial difficulty
 const EXPECTED_MAKER_HASHRATE = 50000;
 
-function padIndex(number, prefix) {
+/**
+ * Given an index (non-negative integer), return it as a
+ * hex string left-padded to 32 chars. Pass prefix boolean
+ * as `true` to get an '0x' prefix.
+ * @param {number} index 
+ * @param {boolean} prefix 
+ */
+function padIndex(index, prefix) {
   if(prefix) {
-    return utils.addHexPrefix(utils.setLengthLeft([number], 32, false).toString('hex'));
+    return utils.addHexPrefix(utils.setLengthLeft([index], 32, false).toString('hex'));
   }
-  return utils.setLengthLeft([number], 32, false).toString('hex');
+  return utils.setLengthLeft([index], 32, false).toString('hex');
 }
 
+/**
+ * Given an Ethereum address (40 characters after removing '0x'),
+ * left-pad with '0's to get a 64-char un-prefixed hex string.
+ * @param {string} address 
+ */
 function padAddress(address) {
   return "000000000000000000000000" + utils.stripHexPrefix(address);
 }
 
+/**
+ * Given a Mapping variable's index and a key to store, return
+ * an '0x'-prefixed hex string for use in the "storage" object
+ * @param {number} index 
+ * @param {string} address 
+ */
 function storageKey(index, address) {
   let paddedAddress = padAddress(address);
   let paddedIndex = padIndex(index);
   let result = utils.sha3(new Buffer(paddedAddress+paddedIndex, 'hex')).toString('hex');
   return utils.addHexPrefix(result)
 }
 
+/**
+ * Given a contract address, a variable index, and an array of
+ * addresses, add all of the key-value pairs so each address
+ * has a `true` value in that map.  Globally modifies the
+ * `template` object.
+ * 
+ * @param {*} alloc 
+ * @param {*} index 
+ * @param {*} addresses 
+ */
 function mapAddressesAt(alloc, index, addresses) {
   let value = '0x01';
   for(let i=0; i<addresses.length; i++) {
@@ -45,21 +73,43 @@ function mapAddressesAt(alloc, index, addresses) {
   }
 }
 
-function buildBlockVotingStorage(input) {
+/**
+ * Configure the BlockVoting contract. In the POW model,
+ * the voting threshold is 0.  We use the `voters` key to set
+ * `voterCount` & `canVote`, then the `makers` key to set
+ * `canCreateBlocks` and `blockMakerCount`.
+ * @param {*} config 
+ */
+function buildBlockVotingStorage(config) {
   template['alloc'][BV_ADDR].storage[padIndex(1,true)] = utils.addHexPrefix(utils.setLengthLeft([0], 1, false).toString('hex'));
-  template['alloc'][BV_ADDR].storage[padIndex(2,true)] = utils.addHexPrefix(utils.setLengthLeft([input.voters.length], 1, false).toString('hex'));
-  mapAddressesAt(BV_ADDR, 3, input.voters);
-  template['alloc'][BV_ADDR].storage[padIndex(4,true)] = utils.addHexPrefix(utils.setLengthLeft([input.makers.length], 1, false).toString('hex'));
-  mapAddressesAt(BV_ADDR, 5,input.makers);
+  template['alloc'][BV_ADDR].storage[padIndex(2,true)] = utils.addHexPrefix(utils.setLengthLeft([config.voters.length], 1, false).toString('hex'));
+  mapAddressesAt(BV_ADDR, 3, config.voters);
+  template['alloc'][BV_ADDR].storage[padIndex(4,true)] = utils.addHexPrefix(utils.setLengthLeft([config.makers.length], 1, false).toString('hex'));
+  mapAddressesAt(BV_ADDR, 5,config.makers);
 }
 
-function buildGovernanceStorage(input){
-  const govOwners = input.voters.concat([REMAINDER_ADDR]);
+/**
+ * Set initial values for `owners` and `numOwners` in the
+ * `WeylGovDeployable.sol` contract.  The contract is owned
+ * by all of the addresses in the config's `voter` key, as 
+ * well as the REMAINDER_ADDR which stores all excess EXC.
+ * @param {*} config 
+ */
+function buildGovernanceStorage(config){
+  const govOwners = config.voters.concat([REMAINDER_ADDR]);
   mapAddressesAt(WEYL_ADDR, 0, govOwners);
   template['alloc'][WEYL_ADDR].storage[padIndex(1, true)] = utils.addHexPrefix(utils.setLengthLeft([govOwners.length], 1, false).toString('hex'));
 }
 
-function fundAddresses(input) {
+/**
+ * Must be called after setting all initial balances.
+ * 
+ * Starting from a max initial supply of 150000000 EXC,
+ * go through every address in initial storage and subtract
+ * its balance from the initial total. Finally, give the
+ * remaining value to the REMAINDER_ADDR above.
+ */
+function fundAddresses() {
   let allocatedBalance = new BigNum(0);
   template['alloc'][BV_ADDR].balance = '0';
   template['alloc'][WEYL_ADDR].balance = '0'
@@ -72,19 +122,36 @@ function fundAddresses(input) {
   template.alloc[REMAINDER_ADDR] = { balance : remainderVal.toString(10) };
 }
 
-function setGasLimit(input) {
-    template['gasLimit'] = input.gasLimit;
+/**
+ * Set the template's `gasLimit` to whatever is included in the config.
+ * @param {*} config 
+ */
+function setGasLimit(config) {
+    template['gasLimit'] = config.gasLimit;
 }
 
-function setDifficulty(input) {
+/**
+ * Given the block config, uses the number of makers to calculate
+ * a difficulty for a desired 10 seconds per block.  Uses the
+ * `EXPECTED_MAKER_HASHRATE` constant to calculate this estimate.
+ * @param {*} config 
+ */
+function setDifficulty(config) {
   let desiredSecondsPerBlock = 10;
-  template['difficulty'] = utils.intToHex(EXPECTED_MAKER_HASHRATE * desiredSecondsPerBlock * input.makers.length);
+  template['difficulty'] = utils.intToHex(EXPECTED_MAKER_HASHRATE * desiredSecondsPerBlock * config.makers.length);
 }
 
-function setChainID(input) {
-  template['config']['chainID'] = input.chainID;
+/**
+ * Sets the template's `chainID` to whatever is included in the config
+ * @param {*} config 
+ */
+function setChainID(config) {
+  template['config']['chainID'] = config.chainID;
 }
 
+/**
+ * Parse and return the genesis block config object.
+ */
 function loadConfig() {
   let fn = path.join(process.cwd(),CONFIG_FILENAME);
   if(!fs.existsSync(fn)) {
@@ -119,13 +186,13 @@ function loadConfig() {
 }
 
 function main() {
-  let input = merge(loadConfig(), saleAddresses);
-  buildBlockVotingStorage(input);
-  buildGovernanceStorage(input);
-  setGasLimit(input);
-  setDifficulty(input);
-  setChainID(input);
-  fundAddresses(input)
+  let config = merge(loadConfig(), saleAddresses);
+  buildBlockVotingStorage(config);
+  buildGovernanceStorage(config);
+  setGasLimit(config);
+  setDifficulty(config);
+  setChainID(config);
+  fundAddresses()
   fs.writeFileSync(path.join(process.cwd(),OUTPUT), JSON.stringify(template, null, 2));
 }
 

package.json

@@ -7,7 +7,8 @@
     "quorum-genesis": "./index.js"
   },
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "node index.js"
   },
   "author": "Dave Bryson",
   "license": "ISC",