Merge upstream 17 6 v1

.gitignore

@@ -5,7 +5,7 @@ bin
 dump
 unit-test
 tmp
-# frontend
+frontend
 .DS_Store
 
 genhtml/

GNUmakefile

@@ -70,9 +70,6 @@ ifndef MACHINE
 MACHINE=native
 endif
 
-$(info Using MACHINE=$(MACHINE))
-$(info Using EXTRAS=$(EXTRAS_PRE) $(EXTRAS))
-
 # Default target
 all:
 

activate

@@ -0,0 +1,85 @@
+for arg in "$@"; do
+  case "$arg" in
+    help|--help)
+      echo "Usage: source activate [CC=clang] [EXTRAS=...] [MACHINE=...] [BUILDDIR=...]"
+      exit 0
+      ;;
+  esac
+done
+
+# Ignore variables set in existing environment.
+unset BUILDDIR1
+unset BUILDDIR
+unset OBJDIR
+unset MACHINE
+unset EXTRAS
+unset CC
+
+# Auto-derive environment variables from the above 5 input variables.
+_fd_env_make_env=$(make env "$@") || return
+eval "$_fd_env_make_env"
+unset _fd_env_make_env
+if [ -z "${OBJDIR+x}" ] || [ -z "$OBJDIR" ]; then
+  printf '%s\n' 'activate: OBJDIR is not set' >&2
+  return
+fi
+
+# Re-export, so future `make` invocations match.
+echo "Firedancer build directory is $OBJDIR"
+export OBJDIR
+export MACHINE
+export EXTRAS
+export CC
+BUILDDIR1="$BUILDDIR"
+export BUILDDIR1
+
+# Update PATH to exclude stale build dirs,
+# and include OBJDIR bin, unit-test, and fuzz-test
+
+_fd_env_script=${BASH_SOURCE[0]:-$0}
+_fd_env_script_dir=$(CDPATH= cd -- "$(dirname -- "$_fd_env_script")" 2>/dev/null && pwd -P)
+_fd_env_root=$_fd_env_script_dir
+
+_fd_env_old_path=${PATH-}
+_fd_env_new_path=
+
+while [ -n "$_fd_env_old_path" ]; do
+  case $_fd_env_old_path in
+    *:*)
+      _fd_env_path_entry=${_fd_env_old_path%%:*}
+      _fd_env_old_path=${_fd_env_old_path#*:}
+      ;;
+    *)
+      _fd_env_path_entry=$_fd_env_old_path
+      _fd_env_old_path=
+      ;;
+  esac
+
+  case $_fd_env_path_entry in
+    /*) _fd_env_path_match=$_fd_env_path_entry ;;
+    *)  _fd_env_path_match=$PWD/$_fd_env_path_entry ;;
+  esac
+
+  if [ -d "$_fd_env_path_match" ]; then
+    _fd_env_path_match=$(CDPATH= cd -- "$_fd_env_path_match" 2>/dev/null && pwd -P)
+  fi
+
+  case $_fd_env_path_match in
+    "$_fd_env_root"| "$_fd_env_root"/*) continue ;;
+  esac
+
+  if [ -z "$_fd_env_new_path" ]; then
+    _fd_env_new_path=$_fd_env_path_entry
+  else
+    _fd_env_new_path=$_fd_env_new_path:$_fd_env_path_entry
+  fi
+done
+
+PATH=$OBJDIR/bin:$OBJDIR/fuzz-test:$OBJDIR/unit-test
+if [ -n "$_fd_env_new_path" ]; then
+  PATH=$PATH:$_fd_env_new_path
+fi
+export PATH
+
+unset _fd_env_old_path _fd_env_new_path _fd_env_path_entry
+unset _fd_env_path_match _fd_env_root _fd_env_script _fd_env_script_dir

book/api/websocket.md

@@ -535,6 +535,9 @@ Some interesting transitions are,
         "loading_full_snapshot_decompress_bytes_compressed": "826495323",
         "loading_full_snapshot_insert_bytes_decompressed": "4864409599",
         "loading_full_snapshot_insert_accounts": 10634591,
+        "loading_full_snapshot_snapwr_in_bytes_decompressed": "4864409599",
+        "loading_full_snapshot_snapwr_out_bytes_decompressed": "4892160000",
+        "loading_full_snapshot_snapwr_accounts": 10634591,
         "loading_incremental_snapshot_elapsed_seconds": null,
         "loading_incremental_snapshot_reset_count": null,
         "loading_incremental_snapshot_slot": null,
@@ -545,6 +548,9 @@ Some interesting transitions are,
         "loading_incremental_snapshot_decompress_bytes_compressed": null,
         "loading_incremental_snapshot_insert_bytes_decompressed": null,
         "loading_incremental_snapshot_insert_accounts": null,
+        "loading_incremental_snapshot_snapwr_in_bytes_decompressed": null,
+        "loading_incremental_snapshot_snapwr_out_bytes_decompressed": null,
+        "loading_incremental_snapshot_snapwr_accounts": null,
         "wait_for_supermajority_bank_hash": "2CeCyRoYmcctDmbXWrSUfTT4aQkGVCnArAmbdmQ5dGFi",
         "wait_for_supermajority_shred_version": "37500",
         "wait_for_supermajority_attempt": 1,
@@ -574,7 +580,10 @@ Some interesting transitions are,
 | loading_{full\|incremental}_snapshot_decompress_bytes_decompressed    | `number\|null`  | If the phase is at least `loading_{full\|incremental}_snapshot`, this is the (decompressed) number of bytes processed by decompress from the snapshot so far. Otherwise, `null` |
 | loading_{full\|incremental}_snapshot_decompress_bytes_compressed      | `number\|null`  | If the phase is at least `loading_{full\|incremental}_snapshot`, this is the (compressed) number of bytes processed by decompress from the snapshot so far. Otherwise, `null` |
 | loading_{full\|incremental}_snapshot_insert_bytes_decompressed        | `number\|null`  | If the phase is at least `loading_{full\|incremental}_snapshot`, this is the (decompressed) number of bytes processed from the snapshot by the snapshot insert time so far. Otherwise, `null` |
-| loading_{full\|incremental}_snapshot_insert_accounts                  | `number\|null`  | If the phase is at least `loading_{full\|incremental}_snapshot`, this is the current number of inserted accounts from the snapshot into the validator's accounts database. Otherwise, `null` |
+| loading_{full\|incremental}_snapshot_insert_accounts                  | `number\|null`  | If the phase is at least `loading_{full\|incremental}_snapshot`, this is the current number of accounts inserted into the validator's accounts database from this snapshot. Otherwise, `null` |
+| loading_{full\|incremental}_snapshot_snapwr_in_bytes_decompressed     | `number\|null`  | If the phase is at least `loading_{full\|incremental}_snapshot`, this is the (decompressed) number of bytes consumed from the snapshot by the snapshot write (snapwr) stage so far. Otherwise, `null` |
+| loading_{full\|incremental}_snapshot_snapwr_out_bytes_decompressed    | `number\|null`  | If the phase is at least `loading_{full\|incremental}_snapshot`, this is the number of bytes written to the on-disk account database by the snapshot write (snapwr) stage for this snapshot so far. Otherwise, `null` |
+| loading_{full\|incremental}_snapshot_snapwr_accounts                  | `number\|null`  | If the phase is at least `loading_{full\|incremental}_snapshot`, this is the current number of accounts written to the on-disk account database by the snapshot write (snapwr) stage for this snapshot so far. Otherwise, `null` |
 | wait_for_supermajority_bank_hash                                      | `string\|null`  | If the client was configured to include the `waiting_for_supermajority` phase at startup, this is the expected bank hash of the snapshot bank.  This ensures all validators join the cluster with the same starting state. `null` if wait for supermajority is not enabled |
 | wait_for_supermajority_shred_version                                  | `string\|null`  | If the client was configured to include the `waiting_for_supermajority` phase at startup, this is the expected shred version it was configured with.  Shred version is functionally a hash of (genesis_hash, cluster_restart_history) which ensures only nodes which explicitly agree on the restart slot and restart attempt count can communicate with each other. `null` if wait for supermajority is not configured |
 | wait_for_supermajority_attempt                                        | `number\|null`  | If the client was configured to include the `waiting_for_supermajority` phase at startup, this is the number of times this cluster has been restarted onto the snapshot slot, including the current attempt. `null` if wait for supermajority is not configured |
@@ -901,6 +910,7 @@ uses to communicate with the internet.
     "gossip",
     "tpu",
     "repair",
+    "rserve",
     "metrics"
 ]
 ```
@@ -914,6 +924,9 @@ in a client used to consume and forward incoming Solana transactions for
 their next leader slot.
 - repair: a client subsystem which requests any missing block data
 needed by the replay pipeline which may have been lost over the network
+- rserve: "repair serve", a client subsystem which serves repair
+requests from other nodes, responding with any block data they are
+missing
 - metrics: refers to the Firedancer metrics tile, which serves an http
 Prometheus metrics endpoint
 
@@ -922,10 +935,10 @@ Prometheus metrics endpoint
     "topic": "summary",
     "key": "live_network_metrics",
     "value": {
-        "ingress": [12345432, 5431234, 92345, ...],
-        "egress": [12345432, 5431234, 92345, ...],
-        "ingress_ema": [1234543.00, 543123.00, 9234.00, ...],
-        "egress_ema": [1234543.00, 543123.00, 9234.00, ...],
+        "ingress": [12345432, 5431234, 92345, 43210, 8765, ...],
+        "egress": [12345432, 5431234, 92345, 43210, 8765, ...],
+        "ingress_ema": [1234543.00, 543123.00, 9234.00, 4321.00, 876.00, ...],
+        "egress_ema": [1234543.00, 543123.00, 9234.00, 4321.00, 876.00, ...],
         "ingress_max_5m": 15000000,
         "egress_max_5m": 14500000,
     }

config/base.mk

@@ -1,4 +1,7 @@
 BASEDIR?=build
+ifneq ($(BUILDDIR1),)
+BUILDDIR:=$(BUILDDIR1)
+endif
 
 VERBOSE?=0
 OPT?=opt