From aace3f42f8a5b2986fdef7841fcb0ce25e9daaf7 Mon Sep 17 00:00:00 2001 From: abje Date: Thu, 11 May 2023 14:30:35 +0200 Subject: [PATCH 01/25] added fuzz/fuzz.proto to the Makefile --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 938bb314b..8085b96f6 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,9 @@ proto_src := internal/proto/clientpb/client.proto \ internal/proto/hotstuffpb/hotstuff.proto \ internal/proto/orchestrationpb/orchestration.proto \ internal/proto/handelpb/handel.proto \ - metrics/types/types.proto + metrics/types/types.proto \ + fuzz/fuzz.proto + proto_go := $(proto_src:%.proto=%.pb.go) gorums_go := internal/proto/clientpb/client_gorums.pb.go \ internal/proto/hotstuffpb/hotstuff_gorums.pb.go \ From e977f2f73c176e3cbbc7d4d77de407659210d4f7 Mon Sep 17 00:00:00 2001 From: abje Date: Thu, 11 May 2023 14:31:19 +0200 Subject: [PATCH 02/25] initial commit --- fuzz/convert.go | 1 + fuzz/error.go | 182 +++++++++++++ fuzz/experimental_string.go | 143 ++++++++++ fuzz/extra_test.go | 82 ++++++ fuzz/fuzz.go | 74 +++++ fuzz/fuzz.proto | 37 +++ fuzz/fuzz_test.go | 162 +++++++++++ fuzz/network.go | 519 ++++++++++++++++++++++++++++++++++++ fuzz/scenario.go | 221 +++++++++++++++ fuzz/serialize.go | 128 +++++++++ fuzz/tostring.go | 372 ++++++++++++++++++++++++++ fuzz/types.go | 63 +++++ 12 files changed, 1984 insertions(+) create mode 100644 fuzz/convert.go create mode 100644 fuzz/error.go create mode 100644 fuzz/experimental_string.go create mode 100644 fuzz/extra_test.go create mode 100644 fuzz/fuzz.go create mode 100644 fuzz/fuzz.proto create mode 100644 fuzz/fuzz_test.go create mode 100644 fuzz/network.go create mode 100644 fuzz/scenario.go create mode 100644 fuzz/serialize.go create mode 100644 fuzz/tostring.go create mode 100644 fuzz/types.go diff --git a/fuzz/convert.go b/fuzz/convert.go new file mode 100644 index 000000000..504e8e0a0 --- /dev/null +++ b/fuzz/convert.go @@ -0,0 +1 @@ +package fuzz diff --git a/fuzz/error.go b/fuzz/error.go new file mode 100644 index 000000000..24da7345a --- /dev/null +++ b/fuzz/error.go @@ -0,0 +1,182 @@ +package fuzz + +import ( + "fmt" + reflect "reflect" + "sort" + "strconv" + "strings" + "testing" +) + +type TypeCount map[reflect.Type]int + +func (typeCount TypeCount) Add(typ reflect.Type) { + typeCount[typ]++ +} + +func (typeCount TypeCount) String(typeTotalCount TypeCount) string { + keys := make([]reflect.Type, 0) + for key := range typeCount { + keys = append(keys, key) + } + + sort.Slice(keys, func(i, j int) bool { return keys[i].String() < keys[j].String() }) + + str := "" + for _, key := range keys { + str += key.String() + ": " + strconv.Itoa(typeCount[key]) + " / " + strconv.Itoa(typeTotalCount[key]) + "\n" + } + return str +} + +type PanicInfo struct { + Err any + StackTrace string + FuzzMsg string + FuzzMsgB64 string + Seed *int64 + LineNum int + TypeCount TypeCount +} + +type ErrorInfo struct { + messageFile string + currentFuzzMsg *FuzzMsg + currentFuzzMsgB64 string + currentFuzzMsgSeed *int64 + errorCount int + panics map[string]PanicInfo + totalScenarios int + failedScenarios int + totalMessages int + failedMessages int + TypePanicCount TypeCount + TypeTotalCount TypeCount +} + +func (errorInfo *ErrorInfo) Init() { + errorInfo.panics = make(map[string]PanicInfo) + errorInfo.TypeTotalCount = make(TypeCount) + errorInfo.TypePanicCount = make(TypeCount) +} + +func (errorInfo *ErrorInfo) OutputInfo(t *testing.T) { + + b64s := "" + seeds := "" + + fmt.Println() + fmt.Println() + fmt.Println() + + fmt.Println("ERROR INFO") + + keys := make([]string, 0) + for key := range errorInfo.panics { + keys = append(keys, key) + } + + //sorting the keys of the + sort.Strings(keys) + + for i, key := range keys { + panicInfo := errorInfo.panics[key] + b64s += panicInfo.FuzzMsgB64 + "\n" + + if panicInfo.Seed != nil { + seeds += strconv.FormatInt(*panicInfo.Seed, 10) + "\n" + } + + fmt.Println() + fmt.Printf("ERROR NUMBER %d\n", i+1) + //contains error location, err text and recover point + fmt.Println(key) + fmt.Println() + fmt.Println("crash amounts grouped by type:") + fmt.Println(panicInfo.TypeCount.String(errorInfo.TypeTotalCount)) + fmt.Println("- STACK TRACE BEGIN") + fmt.Print(panicInfo.StackTrace) + fmt.Println("- STACK TRACE END") + fmt.Println() + fmt.Println("- FUZZ MESSAGE BEGIN") + fmt.Println(panicInfo.FuzzMsg) + fmt.Println("- FUZZ MESSAGE END") + fmt.Println() + + if t != nil { + t.Error(panicInfo.Err) + } + } + + saveStringToFile("previous_messages.b64", b64s) + + if seeds != "" { + saveStringToFile("previous_messages.seed", seeds) + } + + fmt.Println() + fmt.Println("SUMMARY") + fmt.Printf("unique errors found: %d\n", len(errorInfo.panics)) + fmt.Printf("%d runs were errors\n", errorInfo.errorCount) + fmt.Printf("%d of %d scenarios failed\n", errorInfo.failedScenarios, errorInfo.totalScenarios) + fmt.Printf("%d of %d messages failed\n", errorInfo.failedMessages, errorInfo.totalMessages) + fmt.Println() + fmt.Println("crash amounts grouped by type:") + fmt.Println(errorInfo.TypePanicCount.String(errorInfo.TypeTotalCount)) +} + +func (errorInfo *ErrorInfo) AddTotal(fuzzMessage *FuzzMsg, seed *int64) { + errorInfo.totalMessages++ + errorInfo.currentFuzzMsg = fuzzMessage + errorInfo.currentFuzzMsgSeed = seed + typ := reflect.TypeOf(fuzzMessage.Msg()) + errorInfo.TypeTotalCount.Add(typ) +} + +func (errorInfo *ErrorInfo) AddPanic(fullStack string, err2 any, info string) { + + simpleStack := SimplifyStack(fullStack) + identifier := "error location:\t" + simpleStack + "\nerror info:\t" + fmt.Sprint(err2) + "\nrecovered from:\t" + info + + errorInfo.errorCount++ + + oldPanic, okPanic := errorInfo.panics[identifier] + + b64, err := fuzzMsgToB64(errorInfo.currentFuzzMsg) + if err != nil { + panic(err) + } + + FuzzMsgString := errorInfo.currentFuzzMsg.Msg().ToString(0) + newLines := strings.Count(FuzzMsgString, "\n") + + newPanic := PanicInfo{ + Err: err2, + StackTrace: fullStack, + FuzzMsg: FuzzMsgString, + FuzzMsgB64: b64, + Seed: errorInfo.currentFuzzMsgSeed, + LineNum: newLines, + } + + if okPanic { + newPanic.TypeCount = oldPanic.TypeCount + } else { + newPanic.TypeCount = make(TypeCount) + } + + oldLines := oldPanic.LineNum + if !okPanic || newLines < oldLines { + errorInfo.panics[identifier] = newPanic + } + typ := reflect.TypeOf(errorInfo.currentFuzzMsg.Msg()) + errorInfo.panics[identifier].TypeCount.Add(typ) + errorInfo.TypePanicCount.Add(typ) +} + +func SimplifyStack(stack string) string { + stackLines := strings.Split(strings.ReplaceAll(stack, "\r\n", "\n"), "\n") + // line 9 tells us where the panic happened, found through testing + return stackLines[8][1:] +} diff --git a/fuzz/experimental_string.go b/fuzz/experimental_string.go new file mode 100644 index 000000000..f946a34d9 --- /dev/null +++ b/fuzz/experimental_string.go @@ -0,0 +1,143 @@ +package fuzz + +import ( + "fmt" + + "google.golang.org/protobuf/reflect/protoreflect" +) + +// work in progress +func InterfaceToString(i interface{}) string { + return fmt.Sprintf("%v", i) +} + +func ValueToString(v protoreflect.Value, depth int) string { + _, ok := v.Interface().(protoreflect.Message) + + if ok { + return ProtoToString(v.Message(), depth+1) + } else { + return InterfaceToString(v.Interface()) + } +} + +func ListToString(list protoreflect.List, depth int) string { + + if list.Len() == 0 { + return "[]" + } + + tabs := depthToTabs(depth) + + str := "[\n" + + for i := 0; i < list.Len(); i++ { + item := list.Get(i) + str += tabs + "\t" + ValueToString(item, depth) + ",\n" + } + + str += tabs + "]" + + return str +} + +func MapToString(mp protoreflect.Map, depth int) string { + + if mp.Len() == 0 { + return "[]" + } + + tabs := depthToTabs(depth) + str := "[\n" + + mp.Range(func(mk protoreflect.MapKey, v protoreflect.Value) bool { + str += tabs + "\t" + ValueToString(mk.Value(), depth) + ": " + ValueToString(v, depth) + ",\n" + + return true + }) + + str += tabs + "]" + + return str +} + +func FieldToString(field protoreflect.FieldDescriptor, prm protoreflect.Message, depth int) string { + msg := field.Message() + + str := "" + + if msg == nil { + return str + prm.Get(field).String() + "\n" + } + + if field.IsList() { + list := prm.Get(field).List() + return str + ListToString(list, depth) + "\n" + } + + if field.IsExtension() { + panic("extension to string not implemented") + } + + if field.IsMap() { + map2 := prm.Get(field).Map() + return str + MapToString(map2, depth) + "\n" + } + + if field.IsPlaceholder() { + panic("placeholder to string not implemented") + } + + refl := prm.Get(field).Message() + + if refl == nil { + panic("message reflection is nil") + } + + return str + ProtoToString(refl, depth) + "\n" +} + +func ProtoToString(prm protoreflect.Message, depth int) string { + + if !prm.IsValid() { + return "nil\n" + } + + desc := prm.Descriptor() + tabs := depthToTabs(depth) + str := string(desc.FullName()) + "{\n" + fields := desc.Fields() + + for i := 0; i < fields.Len(); i++ { + field := fields.Get(i) + + if field.ContainingOneof() != nil { + continue + } + + str += tabs + "\t" + string(field.Name()) + ": " + FieldToString(field, prm, depth+1) + } + + oneofs := desc.Oneofs() + + for i := 0; i < oneofs.Len(); i++ { + oneof := oneofs.Get(i) + + fields := oneof.Fields() + + for j := 0; j < fields.Len(); j++ { + field := fields.Get(j) + + refl := prm.Get(field).Message() + if !refl.IsValid() { + continue + } + + str += tabs + "\t" + string(oneof.Name()) + ": " + FieldToString(field, prm, depth+1) + } + } + + str += tabs + "}" + + return str +} diff --git a/fuzz/extra_test.go b/fuzz/extra_test.go new file mode 100644 index 000000000..dd5687452 --- /dev/null +++ b/fuzz/extra_test.go @@ -0,0 +1,82 @@ +package fuzz + +// this file includes a couple of extra unnecessary tests +// used for propability tests, profiling and other things + +import ( + "fmt" + "math/rand" + "testing" + + "google.golang.org/protobuf/proto" +) + +func TestFrequencyErrorFuzz(t *testing.T) { + + frequency := make(map[string]int, 0) + + f := initFuzz() + for j := 0; j < 1000; j++ { + errorInfo := new(ErrorInfo) + errorInfo.Init() + + iterations := 1 + + for i := 0; i < iterations; i++ { + fuzzMessage := createFuzzMessage(f, nil) + useFuzzMessage(errorInfo, fuzzMessage, nil) + } + + for key := range errorInfo.panics { + frequency[key]++ + } + } + + sum := 0 + + for key, val := range frequency { + sum += val + fmt.Println(key) + fmt.Println(val) + fmt.Println() + } + + fmt.Println(sum) + +} + +func BenchmarkFuzz(b *testing.B) { + errorInfo := new(ErrorInfo) + errorInfo.Init() + + f := initFuzz() + + for i := 0; i < b.N; i++ { + seed := rand.Int63() + fuzzMessage := createFuzzMessage(f, &seed) + useFuzzMessage(errorInfo, fuzzMessage, &seed) + } + + fmt.Println(b.Elapsed()) + + //errorInfo.OutputInfo(nil) + + fmt.Println() +} + +func TestExperimentalString(t *testing.T) { + + f := initFuzz() + fuzzMessage := createFuzzMessage(f, nil) + + /*fuzzMessage := hotstuffpb.ECDSASignature{ + Signer: uint32(10), + R: []byte{3, 1, 2}, + S: []byte{5, 4, 3}, + }*/ + + var msg proto.Message + msg = fuzzMessage + + fmt.Println(ProtoToString(msg.ProtoReflect(), 0)) +} diff --git a/fuzz/fuzz.go b/fuzz/fuzz.go new file mode 100644 index 000000000..452430b8d --- /dev/null +++ b/fuzz/fuzz.go @@ -0,0 +1,74 @@ +package fuzz + +import ( + "math/rand" + + fuzz "github.com/google/gofuzz" + hotstuffpb "github.com/relab/hotstuff/internal/proto/hotstuffpb" +) + +func initFuzz() *fuzz.Fuzzer { + nilChance := 0.1 + + f := fuzz.New().NilChance(nilChance).Funcs( + func(m *FuzzMsg, c fuzz.Continue) { + switch c.Intn(4) { + case 0: + msg := FuzzMsg_ProposeMsg{ + ProposeMsg: &ProposeMsg{}, + } + c.Fuzz(msg.ProposeMsg) + m.Message = &msg + case 1: + msg := FuzzMsg_VoteMsg{ + VoteMsg: &VoteMsg{}, + } + c.Fuzz(msg.VoteMsg) + m.Message = &msg + case 2: + msg := FuzzMsg_TimeoutMsg{ + TimeoutMsg: &TimeoutMsg{}, + } + c.Fuzz(msg.TimeoutMsg) + m.Message = &msg + case 3: + msg := FuzzMsg_NewViewMsg{ + NewViewMsg: &NewViewMsg{}, + } + c.Fuzz(msg.NewViewMsg) + m.Message = &msg + } + }, + func(sig **hotstuffpb.QuorumSignature, c fuzz.Continue) { + if c.Float64() < nilChance { + *sig = nil + return + } + + *sig = new(hotstuffpb.QuorumSignature) + switch c.Intn(2) { + case 0: + ecdsa := new(hotstuffpb.QuorumSignature_ECDSASigs) + c.Fuzz(&ecdsa) + (*sig).Sig = ecdsa + case 1: + bls12 := new(hotstuffpb.QuorumSignature_BLS12Sig) + c.Fuzz(&bls12) + (*sig).Sig = bls12 + } + }, + ) + + return f +} + +func createFuzzMessage(f *fuzz.Fuzzer, seed *int64) *FuzzMsg { + + if seed != nil { + f.RandSource(rand.NewSource(*seed)) + } + + newMessage := new(FuzzMsg) + f.Fuzz(newMessage) + return newMessage +} diff --git a/fuzz/fuzz.proto b/fuzz/fuzz.proto new file mode 100644 index 000000000..f87efbf61 --- /dev/null +++ b/fuzz/fuzz.proto @@ -0,0 +1,37 @@ +syntax = "proto3"; + +package fuzz; + +import "internal/proto/hotstuffpb/hotstuff.proto"; + +option go_package = "github.com/relab/hotstuff/fuzz"; + +message FuzzMsg { + oneof Message { + ProposeMsg ProposeMsg = 1; + TimeoutMsg TimeoutMsg = 2; + VoteMsg VoteMsg = 3; + NewViewMsg NewViewMsg = 4; + } +} + +message ProposeMsg { + uint32 ID = 1; + hotstuffpb.Proposal Proposal = 2; +} + +message TimeoutMsg { + uint32 ID = 1; + hotstuffpb.TimeoutMsg TimeoutMsg = 2; +} + +message VoteMsg { + uint32 ID = 1; + hotstuffpb.PartialCert PartialCert = 2; + bool Deferred = 3; +} + +message NewViewMsg { + uint32 ID = 1; + hotstuffpb.SyncInfo SyncInfo = 2; +} \ No newline at end of file diff --git a/fuzz/fuzz_test.go b/fuzz/fuzz_test.go new file mode 100644 index 000000000..97d66a628 --- /dev/null +++ b/fuzz/fuzz_test.go @@ -0,0 +1,162 @@ +package fuzz + +import ( + "fmt" + "math/rand" + "runtime/debug" + "testing" + + _ "github.com/relab/hotstuff/consensus/chainedhotstuff" +) + +func TryExecuteScenario(errorInfo *ErrorInfo, oldMessage any, newMessage any) { + errorInfo.totalScenarios++ + defer func() { + if err := recover(); err != nil { + stack := string(debug.Stack()) + + errorInfo.AddPanic(stack, err, "TryExecuteScenario") + errorInfo.failedScenarios++ + } + }() + + var numNodes uint8 = 4 + + allNodesSet := make(NodeSet) + for i := 1; i <= int(numNodes); i++ { + allNodesSet.Add(uint32(i)) + } + + s := Scenario{} + s = append(s, View{Leader: 1, Partitions: []NodeSet{allNodesSet}}) + s = append(s, View{Leader: 1, Partitions: []NodeSet{allNodesSet}}) + s = append(s, View{Leader: 1, Partitions: []NodeSet{allNodesSet}}) + s = append(s, View{Leader: 1, Partitions: []NodeSet{allNodesSet}}) + + result, err := ExecuteScenario(s, numNodes, 0, 100, "chainedhotstuff", oldMessage, newMessage) + + if err != nil { + panic(err) + } + + if !result.Safe { + panic("Expected no safety violations") + } + + if result.Commits != 1 { + panic(fmt.Sprintf("Expected one commit (got %d)", result.Commits)) + } +} + +func getMessagesBasicScenario() int { + var numNodes uint8 = 4 + + allNodesSet := make(NodeSet) + for i := 1; i <= int(numNodes); i++ { + allNodesSet.Add(uint32(i)) + } + + s := Scenario{} + s = append(s, View{Leader: 1, Partitions: []NodeSet{allNodesSet}}) + s = append(s, View{Leader: 1, Partitions: []NodeSet{allNodesSet}}) + s = append(s, View{Leader: 1, Partitions: []NodeSet{allNodesSet}}) + s = append(s, View{Leader: 1, Partitions: []NodeSet{allNodesSet}}) + + result, _ := ExecuteScenario(s, numNodes, 0, 100, "chainedhotstuff") + + messageCount := result.MessageCount + + return messageCount +} + +func fuzzScenario(errorInfo *ErrorInfo, newMessage any) { + TryExecuteScenario(errorInfo, 1, newMessage) +} + +func fuzzMsgToMsg(errorInfo *ErrorInfo, fuzzMsg *FuzzMsg) any { + errorInfo.totalMessages++ + defer func() { + if err := recover(); err != nil { + stack := string(debug.Stack()) + + errorInfo.AddPanic(stack, err, "fuzzMsgToMsg") + errorInfo.failedMessages++ + } + }() + + return fuzzMsg.Msg().ToMsg() +} + +func useFuzzMessage(errorInfo *ErrorInfo, fuzzMessage *FuzzMsg, seed *int64) { + errorInfo.AddTotal(fuzzMessage, seed) + + newMessage := fuzzMsgToMsg(errorInfo, fuzzMessage) + + if newMessage != nil { + fuzzScenario(errorInfo, newMessage) + } +} + +// the main test +func TestFuzz(t *testing.T) { + errorInfo := new(ErrorInfo) + errorInfo.Init() + + f := initFuzz() + + iterations := 1000 + + for i := 0; i < iterations; i++ { + // \r is carriage return, writing the + // next line will overwrite the previous ;) + fmt.Printf("running test %4d/%4d %4d errors \r", i+1, iterations, errorInfo.errorCount) + + seed := rand.Int63() + fuzzMessage := createFuzzMessage(f, &seed) + useFuzzMessage(errorInfo, fuzzMessage, &seed) + } + + errorInfo.OutputInfo(t) +} + +// load previously created fuzz messages from a file +// it doesn't work quite right, i blame proto.Marshal() +func TestPreviousFuzz(t *testing.T) { + + errorInfo := new(ErrorInfo) + errorInfo.Init() + + fuzzMsgs, err := loadFuzzMessagesFromFile("previous_messages.b64") + + if err != nil { + panic(err) + } + + for _, fuzzMessage := range fuzzMsgs { + useFuzzMessage(errorInfo, fuzzMessage, nil) + } + + errorInfo.OutputInfo(t) +} + +// load previously created fuzz messages from a file +// it recreates the fuzz messages from a 64-bit source +func TestSeedPreviousFuzz(t *testing.T) { + errorInfo := new(ErrorInfo) + errorInfo.Init() + + seeds, err := loadSeedsFromFile("previous_messages.seed") + + if err != nil { + panic(err) + } + + f := initFuzz() + + for _, seed := range seeds { + fuzzMessage := createFuzzMessage(f, &seed) + useFuzzMessage(errorInfo, fuzzMessage, nil) + } + + errorInfo.OutputInfo(t) +} diff --git a/fuzz/network.go b/fuzz/network.go new file mode 100644 index 000000000..5f6433f3a --- /dev/null +++ b/fuzz/network.go @@ -0,0 +1,519 @@ +package fuzz + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + "time" + + "github.com/relab/hotstuff" + "github.com/relab/hotstuff/blockchain" + "github.com/relab/hotstuff/consensus" + "github.com/relab/hotstuff/crypto" + "github.com/relab/hotstuff/crypto/ecdsa" + "github.com/relab/hotstuff/crypto/keygen" + "github.com/relab/hotstuff/eventloop" + "github.com/relab/hotstuff/logging" + "github.com/relab/hotstuff/modules" + "github.com/relab/hotstuff/synchronizer" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" +) + +// NodeID is an ID that is unique to a node in the network. +// The ReplicaID is the ID that the node uses when taking part in the consensus protocol, +// while the NetworkID is used to distinguish nodes on the network. +type NodeID struct { + ReplicaID hotstuff.ID + NetworkID uint32 +} + +func (id NodeID) String() string { + return fmt.Sprintf("r%dn%d", id.ReplicaID, id.NetworkID) +} + +type node struct { + blockChain modules.BlockChain + consensus modules.Consensus + eventLoop *eventloop.EventLoop + leaderRotation modules.LeaderRotation + synchronizer modules.Synchronizer + opts *modules.Options + + id NodeID + executedBlocks []*hotstuff.Block + effectiveView hotstuff.View + log strings.Builder +} + +func (n *node) InitModule(mods *modules.Core) { + mods.Get( + &n.blockChain, + &n.consensus, + &n.eventLoop, + &n.leaderRotation, + &n.synchronizer, + &n.opts, + ) +} + +type pendingMessage struct { + message any + receiver uint32 +} + +// Network is a simulated network that supports twins. +type Network struct { + nodes map[uint32]*node + // Maps a replica ID to a replica and its twins. + replicas map[hotstuff.ID][]*node + // For each view (starting at 1), contains the list of partitions for that view. + views []View + + OldMessage int + NewMessage any + + Messages []any + + MessageCounter int + + // the message types to drop + dropTypes map[reflect.Type]struct{} + + pendingMessages []pendingMessage + + logger logging.Logger + // the destination of the logger + log strings.Builder +} + +// NewSimpleNetwork creates a simple network. +func NewSimpleNetwork() *Network { + return &Network{ + nodes: make(map[uint32]*node), + replicas: make(map[hotstuff.ID][]*node), + dropTypes: make(map[reflect.Type]struct{}), + } +} + +// NewPartitionedNetwork creates a new Network with the specified partitions. +// partitions specifies the network partitions for each view. +func NewPartitionedNetwork(views []View, dropTypes ...any) *Network { + n := &Network{ + nodes: make(map[uint32]*node), + replicas: make(map[hotstuff.ID][]*node), + views: views, + dropTypes: make(map[reflect.Type]struct{}), + } + n.logger = logging.NewWithDest(&n.log, "network") + for _, t := range dropTypes { + n.dropTypes[reflect.TypeOf(t)] = struct{}{} + } + return n +} + +// GetNodeBuilder returns a consensus.Builder instance for a node in the network. +func (n *Network) GetNodeBuilder(id NodeID, pk hotstuff.PrivateKey) modules.Builder { + node := node{ + id: id, + } + n.nodes[id.NetworkID] = &node + n.replicas[id.ReplicaID] = append(n.replicas[id.ReplicaID], &node) + builder := modules.NewBuilder(id.ReplicaID, pk) + // register node as an anonymous module because that allows configuration to obtain it. + builder.Add(&node) + return builder +} + +func (n *Network) createTwinsNodes(nodes []NodeID, scenario Scenario, consensusName string) error { + cg := &commandGenerator{} + for _, nodeID := range nodes { + + var err error + pk, err := keygen.GenerateECDSAPrivateKey() + if err != nil { + return err + } + + builder := n.GetNodeBuilder(nodeID, pk) + node := n.nodes[nodeID.NetworkID] + + consensusModule, ok := modules.GetModule[consensus.Rules](consensusName) + if !ok { + return fmt.Errorf("unknown consensus module: '%s'", consensusName) + } + builder.Add( + eventloop.New(100), + blockchain.New(), + consensus.New(consensusModule), + consensus.NewVotingMachine(), + crypto.NewCache(ecdsa.New(), 100), + synchronizer.New(FixedTimeout(0)), + logging.NewWithDest(&node.log, fmt.Sprintf("r%dn%d", nodeID.ReplicaID, nodeID.NetworkID)), + // twins-specific: + &configuration{network: n, node: node}, + &timeoutManager{network: n, node: node, timeout: 5}, + leaderRotation(n.views), + &commandModule{commandGenerator: cg, node: node}, + ) + builder.Options().SetShouldVerifyVotesSync() + builder.Build() + } + return nil +} + +func (n *Network) run(ticks int) { + // kick off the initial proposal(s) + for _, node := range n.nodes { + if node.leaderRotation.GetLeader(1) == node.id.ReplicaID { + node.consensus.Propose(node.synchronizer.(*synchronizer.Synchronizer).SyncInfo()) + } + } + + for tick := 0; tick < ticks; tick++ { + n.tick() + } +} + +// tick performs one tick for each node +func (n *Network) tick() { + for _, msg := range n.pendingMessages { + n.nodes[msg.receiver].eventLoop.AddEvent(msg.message) + } + n.pendingMessages = nil + + for _, node := range n.nodes { + node.eventLoop.AddEvent(tick{}) + // run each event loop as long as it has events + for node.eventLoop.Tick() { + } + } +} + +// shouldDrop decides if the sender should drop the message, based on the current view of the sender and the +// partitions configured for that view. +func (n *Network) shouldDrop(sender, receiver uint32, message any) bool { + + node, ok := n.nodes[sender] + if !ok { + panic(fmt.Errorf("node matching sender id %d was not found", sender)) + } + + // Index into viewPartitions. + i := -1 + if node.effectiveView > node.synchronizer.View() { + i += int(node.effectiveView) + } else { + i += int(node.synchronizer.View()) + } + + if i < 0 { + return false + } + + // will default to dropping all messages from views that don't have any specified partitions. + if i >= len(n.views) { + return true + } + + partitions := n.views[i].Partitions + for _, partition := range partitions { + if partition.Contains(sender) && partition.Contains(receiver) { + return false + } + } + + _, ok = n.dropTypes[reflect.TypeOf(message)] + + return ok +} + +func (n *Network) shouldSwap(message any) bool { + n.logger.Infof("is %d equal to %d?", n.OldMessage, n.MessageCounter) + return n.OldMessage == n.MessageCounter +} + +// NewConfiguration returns a new Configuration module for this network. +func (n *Network) NewConfiguration() modules.Configuration { + return &configuration{network: n} +} + +type configuration struct { + node *node + network *Network + subConfig hotstuff.IDSet +} + +// alternative way to get a pointer to the node. +func (c *configuration) InitModule(mods *modules.Core) { + if c.node == nil { + mods.TryGet(&c.node) + } +} + +func (c *configuration) broadcastMessage(message any) { + c.network.logger.Infof("broadcasting message") + for id := range c.network.replicas { + if id == c.node.id.ReplicaID { + // do not send message to self or twin + continue + } else if c.subConfig == nil || c.subConfig.Contains(id) { + c.sendMessage(id, message) + } + } +} + +func (c *configuration) sendMessage(id hotstuff.ID, message any) { + + nodes, ok := c.network.replicas[id] + if !ok { + panic(fmt.Errorf("attempt to send message to replica %d, but this replica does not exist", id)) + } + + for _, node := range nodes { + + if c.shouldDrop(node.id, message) { + c.network.logger.Infof("node %v -> node %v: DROP %T(%v)", c.node.id, node.id, message, message) + continue + } + + c.network.Messages = append(c.network.Messages, message) + c.network.MessageCounter += 1 + + if c.network.shouldSwap(message) { + + /*if (c.network.NewMessage.(hotstuff.ProposeMsg).Block.Parent() == hotstuff.Hash{}) { + c.network.NewMessage.(hotstuff.ProposeMsg).Block.SetParent(message.(hotstuff.ProposeMsg).Block.Parent()) + }*/ + c.network.logger.Infof("swapping message with fuzz message") + message = c.network.NewMessage + } + + c.network.logger.Infof("node %v -> node %v: SEND %T(%v)", c.node.id, node.id, message, message) + c.network.pendingMessages = append( + c.network.pendingMessages, + pendingMessage{ + receiver: uint32(node.id.NetworkID), + message: message, + }, + ) + } +} + +// shouldDrop checks if a message to the node identified by id should be dropped. +func (c *configuration) shouldDrop(id NodeID, message any) bool { + // retrieve the drop config for this node. + return c.network.shouldDrop(c.node.id.NetworkID, id.NetworkID, message) +} + +// Replicas returns all of the replicas in the configuration. +func (c *configuration) Replicas() map[hotstuff.ID]modules.Replica { + m := make(map[hotstuff.ID]modules.Replica) + for id := range c.network.replicas { + m[id] = &replica{ + config: c, + id: id, + } + } + return m +} + +// Replica returns a replica if present in the configuration. +func (c *configuration) Replica(id hotstuff.ID) (r modules.Replica, ok bool) { + if _, ok = c.network.replicas[id]; ok { + return &replica{ + config: c, + id: id, + }, true + } + return nil, false +} + +// SubConfig returns a subconfiguration containing the replicas specified in the ids slice. +func (c *configuration) SubConfig(ids []hotstuff.ID) (sub modules.Configuration, err error) { + subConfig := hotstuff.NewIDSet() + for _, id := range ids { + subConfig.Add(id) + } + return &configuration{ + node: c.node, + network: c.network, + subConfig: subConfig, + }, nil +} + +// Len returns the number of replicas in the configuration. +func (c *configuration) Len() int { + return len(c.network.replicas) +} + +// QuorumSize returns the size of a quorum. +func (c *configuration) QuorumSize() int { + return hotstuff.QuorumSize(c.Len()) +} + +// Propose sends the block to all replicas in the configuration. +func (c *configuration) Propose(proposal hotstuff.ProposeMsg) { + c.broadcastMessage(proposal) +} + +// Timeout sends the timeout message to all replicas. +func (c *configuration) Timeout(msg hotstuff.TimeoutMsg) { + c.broadcastMessage(msg) +} + +// Fetch requests a block from all the replicas in the configuration. +func (c *configuration) Fetch(_ context.Context, hash hotstuff.Hash) (block *hotstuff.Block, ok bool) { + for _, replica := range c.network.replicas { + for _, node := range replica { + if c.shouldDrop(node.id, hash) { + continue + } + block, ok = node.blockChain.LocalGet(hash) + if ok { + return block, true + } + } + } + return nil, false +} + +type replica struct { + // pointer to the node that wants to contact this replica. + config *configuration + // id of the replica. + id hotstuff.ID +} + +// ID returns the replica's id. +func (r *replica) ID() hotstuff.ID { + return r.config.network.replicas[r.id][0].id.ReplicaID +} + +// PublicKey returns the replica's public key. +func (r *replica) PublicKey() hotstuff.PublicKey { + return r.config.network.replicas[r.id][0].opts.PrivateKey().Public() +} + +// Vote sends the partial certificate to the other replica. +func (r *replica) Vote(cert hotstuff.PartialCert) { + r.config.sendMessage(r.id, hotstuff.VoteMsg{ + ID: r.config.node.opts.ID(), + PartialCert: cert, + }) +} + +// NewView sends the quorum certificate to the other replica. +func (r *replica) NewView(si hotstuff.SyncInfo) { + r.config.sendMessage(r.id, hotstuff.NewViewMsg{ + ID: r.config.node.opts.ID(), + SyncInfo: si, + }) +} + +func (r *replica) Metadata() map[string]string { + return r.config.network.replicas[r.id][0].opts.ConnectionMetadata() +} + +// NodeSet is a set of network ids. +type NodeSet map[uint32]struct{} + +// Add adds a NodeID to the set. +func (s NodeSet) Add(v uint32) { + s[v] = struct{}{} +} + +// Contains returns true if the set contains the NodeID, false otherwise. +func (s NodeSet) Contains(v uint32) bool { + _, ok := s[v] + return ok +} + +// MarshalJSON returns a JSON representation of the node set. +func (s NodeSet) MarshalJSON() ([]byte, error) { + ids := maps.Keys(s) + slices.Sort(ids) + return json.Marshal(ids) +} + +// UnmarshalJSON restores the node set from JSON. +func (s *NodeSet) UnmarshalJSON(data []byte) error { + if *s == nil { + *s = make(NodeSet) + } + var nodes []uint32 + err := json.Unmarshal(data, &nodes) + if err != nil { + return err + } + for _, node := range nodes { + s.Add(node) + } + return nil +} + +type tick struct{} + +type timeoutManager struct { + synchronizer modules.Synchronizer + eventLoop *eventloop.EventLoop + + node *node + network *Network + countdown int + timeout int +} + +func (tm *timeoutManager) advance() { + tm.countdown-- + if tm.countdown == 0 { + view := tm.synchronizer.View() + tm.eventLoop.AddEvent(synchronizer.TimeoutEvent{View: view}) + tm.countdown = tm.timeout + if tm.node.effectiveView <= view { + tm.node.effectiveView = view + 1 + tm.network.logger.Infof("node %v effective view is %d due to timeout", tm.node.id, tm.node.effectiveView) + } + } +} + +func (tm *timeoutManager) viewChange(event synchronizer.ViewChangeEvent) { + tm.countdown = tm.timeout + if event.Timeout { + tm.network.logger.Infof("node %v entered view %d after timeout", tm.node.id, event.View) + } else { + tm.network.logger.Infof("node %v entered view %d after voting", tm.node.id, event.View) + } +} + +// InitModule gives the module a reference to the Modules object. +// It also allows the module to set module options using the OptionsBuilder. +func (tm *timeoutManager) InitModule(mods *modules.Core) { + mods.Get( + &tm.synchronizer, + &tm.eventLoop, + ) + + tm.eventLoop.RegisterObserver(tick{}, func(event any) { + tm.advance() + }) + tm.eventLoop.RegisterObserver(synchronizer.ViewChangeEvent{}, func(event any) { + tm.viewChange(event.(synchronizer.ViewChangeEvent)) + }) +} + +// FixedTimeout returns an ExponentialTimeout with a max exponent of 0. +func FixedTimeout(timeout time.Duration) synchronizer.ViewDuration { + return fixedDuration{timeout} +} + +type fixedDuration struct { + timeout time.Duration +} + +func (d fixedDuration) Duration() time.Duration { return d.timeout } +func (d fixedDuration) ViewStarted() {} +func (d fixedDuration) ViewSucceeded() {} +func (d fixedDuration) ViewTimeout() {} diff --git a/fuzz/scenario.go b/fuzz/scenario.go new file mode 100644 index 000000000..9e6a05d51 --- /dev/null +++ b/fuzz/scenario.go @@ -0,0 +1,221 @@ +package fuzz + +import ( + "context" + "fmt" + "strconv" + "strings" + "sync" + + "github.com/relab/hotstuff" +) + +// View specifies the leader id and the partition scenario for a single view. +type View struct { + Leader hotstuff.ID `json:"leader"` + Partitions []NodeSet `json:"partitions"` +} + +// Scenario specifies the nodes, partitions and leaders for a twins scenario. +type Scenario []View + +func (s Scenario) String() string { + var sb strings.Builder + for i := 0; i < len(s); i++ { + sb.WriteString(fmt.Sprintf("leader: %d, partitions: ", s[i].Leader)) + for _, partition := range s[i].Partitions { + sb.WriteString("[ ") + for id := range partition { + sb.WriteString(fmt.Sprint(id)) + sb.WriteString(" ") + } + sb.WriteString("] ") + } + sb.WriteString("\n") + } + return sb.String() +} + +// ScenarioResult contains the result and logs from executing a scenario. +type ScenarioResult struct { + Safe bool + Commits int + NetworkLog string + NodeLogs map[NodeID]string + NodeCommits map[NodeID][]*hotstuff.Block + Messages []any + MessageCount int +} + +func assignNodeIDs(numNodes, numTwins uint8) (nodes, twins []NodeID) { + replicaID := hotstuff.ID(1) + networkID := uint32(1) + remainingTwins := numTwins + + // assign IDs to nodes + for i := uint8(0); i < numNodes; i++ { + if remainingTwins > 0 { + twins = append(twins, NodeID{ + ReplicaID: replicaID, + NetworkID: networkID, + }) + networkID++ + twins = append(twins, NodeID{ + ReplicaID: replicaID, + NetworkID: networkID, + }) + remainingTwins-- + } else { + nodes = append(nodes, NodeID{ + ReplicaID: replicaID, + NetworkID: networkID, + }) + } + networkID++ + replicaID++ + } + + return +} + +// ExecuteScenario executes a twins scenario. +func ExecuteScenario(scenario Scenario, numNodes, numTwins uint8, numTicks int, consensusName string, replaceMessage ...any) (result ScenarioResult, err error) { + // Network simulator that blocks proposals, votes, and fetch requests between nodes that are in different partitions. + // Timeout and NewView messages are permitted. + + network := NewPartitionedNetwork(scenario, + hotstuff.ProposeMsg{}, + hotstuff.VoteMsg{}, + hotstuff.Hash{}, + hotstuff.NewViewMsg{}, + hotstuff.TimeoutMsg{}, + ) + + if len(replaceMessage) == 2 { + network.OldMessage = replaceMessage[0].(int) + network.NewMessage = replaceMessage[1] + } + + nodes, twins := assignNodeIDs(numNodes, numTwins) + nodes = append(nodes, twins...) + + err = network.createTwinsNodes(nodes, scenario, consensusName) + if err != nil { + return ScenarioResult{}, err + } + + network.run(numTicks) + + nodeLogs := make(map[NodeID]string) + for _, node := range network.nodes { + nodeLogs[node.id] = node.log.String() + } + + // check if the majority of replicas have committed the same blocks + safe, commits := checkCommits(network) + + return ScenarioResult{ + Safe: safe, + Commits: commits, + NetworkLog: network.log.String(), + NodeLogs: nodeLogs, + NodeCommits: getBlocks(network), + Messages: network.Messages, + MessageCount: network.MessageCounter, + }, nil +} + +func checkCommits(network *Network) (safe bool, commits int) { + i := 0 + for { + noCommits := true + commitCount := make(map[hotstuff.Hash]int) + for _, replica := range network.replicas { + if len(replica) != 1 { + // TODO: should we be skipping replicas with twins? + continue + } + if len(replica[0].executedBlocks) <= i { + continue + } + commitCount[replica[0].executedBlocks[i].Hash()]++ + noCommits = false + } + + if noCommits { + break + } + + // if all correct replicas have executed the same blocks, then there should be only one entry in commitCount + // the number of replicas that committed the block could be smaller, if some correct replicas happened to + // be in a different partition at the time when the test ended. + if len(commitCount) != 1 { + return false, i + } + + i++ + } + return true, i +} + +type leaderRotation []View + +// GetLeader returns the id of the leader in the given view. +func (lr leaderRotation) GetLeader(view hotstuff.View) hotstuff.ID { + // we start at view 1 + v := int(view) - 1 + if v >= 0 && v < len(lr) { + return lr[v].Leader + } + // default to 0 (which is an invalid id) + return 0 +} + +func getBlocks(network *Network) map[NodeID][]*hotstuff.Block { + m := make(map[NodeID][]*hotstuff.Block) + for _, node := range network.nodes { + m[node.id] = node.executedBlocks + } + return m +} + +type commandGenerator struct { + mut sync.Mutex + nextCmd uint64 +} + +func (cg *commandGenerator) next() hotstuff.Command { + cg.mut.Lock() + defer cg.mut.Unlock() + cmd := hotstuff.Command(strconv.FormatUint(cg.nextCmd, 10)) + cg.nextCmd++ + return cmd +} + +type commandModule struct { + commandGenerator *commandGenerator + node *node +} + +// Accept returns true if the replica should accept the command, false otherwise. +func (commandModule) Accept(_ hotstuff.Command) bool { + return true +} + +// Proposed tells the acceptor that the propose phase for the given command succeeded, and it should no longer be +// accepted in the future. +func (commandModule) Proposed(_ hotstuff.Command) {} + +// Get returns the next command to be proposed. +// It may run until the context is cancelled. +// If no command is available, the 'ok' return value should be false. +func (cm commandModule) Get(_ context.Context) (cmd hotstuff.Command, ok bool) { + return cm.commandGenerator.next(), true +} + +// Exec executes the given command. +func (cm commandModule) Exec(block *hotstuff.Block) { + cm.node.executedBlocks = append(cm.node.executedBlocks, block) +} + +func (commandModule) Fork(block *hotstuff.Block) {} diff --git a/fuzz/serialize.go b/fuzz/serialize.go new file mode 100644 index 000000000..0129adcde --- /dev/null +++ b/fuzz/serialize.go @@ -0,0 +1,128 @@ +package fuzz + +import ( + "bufio" + "encoding/base64" + "os" + "strconv" + "strings" + + "google.golang.org/protobuf/proto" +) + +func fuzzMsgToB64(msg *FuzzMsg) (string, error) { + bytes, err := proto.Marshal(msg) + str := base64.StdEncoding.EncodeToString(bytes) + + return str, err +} + +func b64ToFuzzMsg(str string) (*FuzzMsg, error) { + bytes, err := base64.StdEncoding.DecodeString(str) + + msg := new(FuzzMsg) + proto.Unmarshal(bytes, msg) + + return msg, err +} + +/*func b64TofuzzMessages(str string) ([]FuzzMsg, error) { + + strLines := strings.Split(strings.ReplaceAll(str, "\r\n", "\n"), "\n") + for _, msg := range messages { + str, err := fuzzMsgToB64(msg) + if err != nil { + return nil, err + } + } + + return full_str, nil +}*/ + +func saveStringToFile(filename string, str string) error { + + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + w := bufio.NewWriter(f) + _, err = w.WriteString(str) + w.Flush() + + return nil +} + +func loadStringFromFile(filename string) (string, error) { + f, err := os.Open(filename) + if err != nil { + return "", err + } + + fi, err := f.Stat() + if err != nil { + return "", err + } + + bytes := make([]byte, fi.Size()) + + f.Read(bytes) + + b64string := string(bytes) + + return b64string, nil +} + +func loadFuzzMessagesFromFile(filename string) ([]*FuzzMsg, error) { + + str, err := loadStringFromFile(filename) + + if err != nil { + return nil, err + } + + b64s := strings.Split(str, "\n") + + fuzzMsgs := make([]*FuzzMsg, 0) + + for _, b64 := range b64s { + + if b64 == "" { + continue + } + + fuzzMsg, err := b64ToFuzzMsg(b64) + if err != nil { + return nil, err + } + fuzzMsgs = append(fuzzMsgs, fuzzMsg) + } + + return fuzzMsgs, nil +} + +func loadSeedsFromFile(filename string) ([]int64, error) { + + str, err := loadStringFromFile(filename) + + if err != nil { + return nil, err + } + + seedstrs := strings.Split(str, "\n") + seeds := make([]int64, 0) + + for _, seedstr := range seedstrs { + if seedstr == "" { + continue + } + + seed, err := strconv.ParseInt(seedstr, 10, 64) + if err != nil { + return nil, err + } + seeds = append(seeds, seed) + } + + return seeds, nil +} diff --git a/fuzz/tostring.go b/fuzz/tostring.go new file mode 100644 index 000000000..58a7c4adc --- /dev/null +++ b/fuzz/tostring.go @@ -0,0 +1,372 @@ +package fuzz + +import ( + "fmt" + + "github.com/relab/hotstuff/internal/proto/hotstuffpb" +) + +func depthToTabs(depth int) (tabs string) { + for i := 0; i < depth; i++ { + tabs += "\t" + } + return +} + +func (proposeFuzzMsg *ProposeMsg) ToString(depth int) string { + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "fuzz.ProposeMsg{\n"+ + "%s\tID: %v\n"+ + "%s\tProposal: %v\n"+ + "%s}", + tabs, proposeFuzzMsg.ID, + tabs, ProposalToString(proposeFuzzMsg.Proposal, depth+1), + tabs) +} + +func (timeoutFuzzMsg *TimeoutMsg) ToString(depth int) string { + + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "fuzz.TimeoutMsg{\n"+ + "%s\tID: %v\n"+ + "%s\tTimeoutMsg: %v\n"+ + "%s}", + tabs, timeoutFuzzMsg.ID, + tabs, TimeoutMsgToString(timeoutFuzzMsg.TimeoutMsg, depth+1), + tabs) +} + +func (voteFuzzMsg *VoteMsg) ToString(depth int) string { + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "fuzz.VoteMsg{\n"+ + "%s\tID: %v\n"+ + "%s\tDeffered: %v\n"+ + "%s\tPartialCert: %v\n"+ + "%s}", + tabs, voteFuzzMsg.ID, + tabs, voteFuzzMsg.Deferred, + tabs, PartialCertToString(voteFuzzMsg.PartialCert, depth+1), + tabs) +} + +func (newViewFuzzMsg *NewViewMsg) ToString(depth int) string { + + if newViewFuzzMsg == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "fuzz.NewViewMsg{\n"+ + "%s\tID: %v\n"+ + "%s\tSyncInfo: %v\n"+ + "%s}", + tabs, newViewFuzzMsg.ID, + tabs, SyncInfoToString(newViewFuzzMsg.SyncInfo, depth+1), + tabs) +} + +func ProposalToString(proposal *hotstuffpb.Proposal, depth int) string { + + if proposal == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "hotstuffpb.Proposal{\n"+ + "%s\tBlock: %v\n"+ + "%s\tAggQC: %v\n"+ + "%s}", + tabs, BlockToString(proposal.Block, depth+1), + tabs, AggQCToString(proposal.AggQC, depth+1), + tabs) +} + +func BlockToString(block *hotstuffpb.Block, depth int) string { + + if block == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "&hotstuffpb.Block{\n"+ + "%s\tParent: %v\n"+ + "%s\tQC: %v\n"+ + "%s\tView: %v\n"+ + "%s\tCommand: %v\n"+ + "%s\tProposer: %v\n"+ + "%s}", + tabs, block.Parent, + tabs, QuorumCertToString(block.QC, depth+1), + tabs, block.View, + tabs, block.Command, + tabs, block.Proposer, + tabs) +} + +func TimeoutMsgToString(timeoutMsg *hotstuffpb.TimeoutMsg, depth int) string { + + if timeoutMsg == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "hotstuffpb.TimeoutMsg{\n"+ + "%s\tView: %v\n"+ + "%s\tSyncInfo: %v\n"+ + "%s\tViewSig: %v\n"+ + "%s\tMsgSig: %v\n"+ + "%s}", + tabs, timeoutMsg.View, + tabs, SyncInfoPtrToString(timeoutMsg.SyncInfo, depth+1), + tabs, QuorumSignatureToString(timeoutMsg.ViewSig, depth+1), + tabs, QuorumSignatureToString(timeoutMsg.MsgSig, depth+1), + tabs) +} + +func SyncInfoPtrToString(object *hotstuffpb.SyncInfo, depth int) string { + if object == nil { + return "nil" + } + + return "&" + SyncInfoToString(object, depth) +} + +func SyncInfoToString(syncInfo *hotstuffpb.SyncInfo, depth int) string { + + if syncInfo == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "hotstuffpb.SyncInfo{\n"+ + "%s\tQC: %v\n"+ + "%s\tTC: %v\n"+ + "%s\tAggQC: %v\n"+ + "%s}", + tabs, QuorumCertToString(syncInfo.QC, depth+1), + tabs, TimeoutCertToString(syncInfo.TC, depth+1), + tabs, AggQCToString(syncInfo.AggQC, depth+1), + tabs) +} + +func QuorumCertToString(qc *hotstuffpb.QuorumCert, depth int) string { + + if qc == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "&hotstuffpb.QuorumCert{\n"+ + "%s\tSig: %v\n"+ + "%s\tView: %v\n"+ + "%s\tHash: %v\n"+ + "%s}", + tabs, QuorumSignatureToString(qc.Sig, depth+1), + tabs, qc.View, + tabs, qc.Hash, + tabs) +} + +func TimeoutCertToString(tc *hotstuffpb.TimeoutCert, depth int) string { + + if tc == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "&hotstuffpb.TimeoutCert{\n"+ + "%s\tSig: %v\n"+ + "%s\tView: %v\n"+ + "%s}", + tabs, QuorumSignatureToString(tc.Sig, depth+1), + tabs, tc.View, + tabs) +} + +func AggQCToString(aggQC *hotstuffpb.AggQC, depth int) string { + + if aggQC == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + QCsString := "map[uint32]*hotstuffpb.QuorumCert{\n" + + for key, value := range aggQC.QCs { + QCsString += fmt.Sprintf("%v\t\t%v: %v\n", tabs, key, QuorumCertToString(value, depth+2)) + } + + QCsString += tabs + "\t" + + return fmt.Sprintf( + "&hotstuffpb.AggQC{\n"+ + "%s\tQCs: %v\n"+ + "%s\tSig: %v\n"+ + "%s\tView: %v\n"+ + "%s}", + tabs, QCsString, + tabs, QuorumSignatureToString(aggQC.Sig, depth+1), + tabs, aggQC.View, + tabs) +} + +func PartialCertToString(partialCert *hotstuffpb.PartialCert, depth int) string { + + if partialCert == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "hotstuffpb.PartialCert{\n"+ + "%s\tSig: %v\n"+ + "%s\tHash: %v\n"+ + "%s}", + tabs, QuorumSignatureToString(partialCert.Sig, depth+1), + tabs, partialCert.Hash, + tabs) +} + +func QuorumSignatureToString(quorumSignature *hotstuffpb.QuorumSignature, depth int) string { + + if quorumSignature == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + sigString := "" + if quorumSignature.Sig == nil { + sigString = "nil" + } else { + switch quorumSignature.Sig.(type) { + case *hotstuffpb.QuorumSignature_ECDSASigs: + sigString = QuorumSignature_ECDSASigsToString(quorumSignature.Sig.(*hotstuffpb.QuorumSignature_ECDSASigs), depth+1) + case *hotstuffpb.QuorumSignature_BLS12Sig: + sigString = QuorumSignature_BLS12SigToString(quorumSignature.Sig.(*hotstuffpb.QuorumSignature_BLS12Sig), depth+1) + } + } + + return fmt.Sprintf( + "&hotstuffpb.QuorumSignature{\n"+ + "%s\tSig: %s\n"+ + "%s}", + tabs, sigString, + tabs) +} + +func QuorumSignature_ECDSASigsToString(ECDASigs *hotstuffpb.QuorumSignature_ECDSASigs, depth int) string { + + if ECDASigs == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "&hotstuffpb.QuorumSignature_ECDSASigs{\n"+ + "%s\tECDASigs: %s\n"+ + "%s},", + tabs, ECDSAMultiSignatureToString(ECDASigs.ECDSASigs, depth+1), + tabs) +} + +func ECDSAMultiSignatureToString(ECDASigs *hotstuffpb.ECDSAMultiSignature, depth int) string { + + if ECDASigs == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + sigsString := "[]*ECDASignature{\n" + + for _, sig := range ECDASigs.Sigs { + sigsString += fmt.Sprintf("%s\t\t%s\n", tabs, ECDASigToString(sig, depth+2)) + } + + sigsString += tabs + "\t}" + + return fmt.Sprintf( + "&hotstuffpb.ECDSAMultiSignature{\n"+ + "%s\tECDASigs: %s\n"+ + "%s}", + tabs, sigsString, + tabs) +} + +func ECDASigToString(sig *hotstuffpb.ECDSASignature, depth int) string { + + if sig == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "&hotstuffpb.ECDSASignature{\n"+ + "%s\tSigner: %v\n"+ + "%s\tR: %v\n"+ + "%s\tS: %v\n"+ + "%s}", + tabs, sig.Signer, + tabs, sig.R, + tabs, sig.S, + tabs) +} + +func QuorumSignature_BLS12SigToString(BLS12Sig *hotstuffpb.QuorumSignature_BLS12Sig, depth int) string { + + if BLS12Sig == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "&hotstuffpb.QuorumSignature_BLS12Sig{\n"+ + "%s\tBLS12Sig: %s\n"+ + "%s},", + tabs, BLS12AggregateSignatureToString(BLS12Sig.BLS12Sig, depth+1), + tabs) +} + +func BLS12AggregateSignatureToString(BLS12Sig *hotstuffpb.BLS12AggregateSignature, depth int) string { + + if BLS12Sig == nil { + return "nil" + } + + tabs := depthToTabs(depth) + + return fmt.Sprintf( + "&hotstuffpb.BLS12AggregateSignature{\n"+ + "%s\tSig: %v\n"+ + "%s\tParticipants: %v\n"+ + "%s},", + tabs, BLS12Sig.Sig, + tabs, BLS12Sig.Participants, + tabs) +} diff --git a/fuzz/types.go b/fuzz/types.go new file mode 100644 index 000000000..d04bc57fa --- /dev/null +++ b/fuzz/types.go @@ -0,0 +1,63 @@ +package fuzz + +import ( + "github.com/relab/hotstuff" + "github.com/relab/hotstuff/internal/proto/hotstuffpb" +) + +type FuzzMsgInterface interface { + ToMsg() any + ToString(int) string + String() string +} + +type AlmostFuzzMsg interface { + Msg() FuzzMsgInterface +} + +func (proposeFuzzMsg *FuzzMsg_ProposeMsg) Msg() FuzzMsgInterface { + return proposeFuzzMsg.ProposeMsg +} + +func (timeoutFuzzMsg *FuzzMsg_TimeoutMsg) Msg() FuzzMsgInterface { + return timeoutFuzzMsg.TimeoutMsg +} + +func (newViewFuzzMsg *FuzzMsg_NewViewMsg) Msg() FuzzMsgInterface { + return newViewFuzzMsg.NewViewMsg +} + +func (voteFuzzMsg *FuzzMsg_VoteMsg) Msg() FuzzMsgInterface { + return voteFuzzMsg.VoteMsg +} + +func (fuzzMsg *FuzzMsg) Msg() FuzzMsgInterface { + return fuzzMsg.Message.(AlmostFuzzMsg).Msg() +} + +func (proposeFuzzMsg *ProposeMsg) ToMsg() any { + proposeMsg := hotstuffpb.ProposalFromProto(proposeFuzzMsg.Proposal) + proposeMsg.ID = hotstuff.ID(proposeFuzzMsg.ID) + return proposeMsg +} + +func (timeoutFuzzMsg *TimeoutMsg) ToMsg() any { + timeoutMsg := hotstuffpb.TimeoutMsgFromProto(timeoutFuzzMsg.TimeoutMsg) + timeoutMsg.ID = hotstuff.ID(timeoutFuzzMsg.ID) + return timeoutMsg +} + +func (voteFuzzMsg *VoteMsg) ToMsg() any { + voteMsg := hotstuff.VoteMsg{} + voteMsg.PartialCert = hotstuffpb.PartialCertFromProto(voteFuzzMsg.PartialCert) + voteMsg.ID = hotstuff.ID(voteFuzzMsg.ID) + voteMsg.Deferred = voteFuzzMsg.Deferred + return voteMsg +} + +func (newViewFuzzMsg *NewViewMsg) ToMsg() any { + newViewMsg := hotstuff.NewViewMsg{} + newViewMsg.SyncInfo = hotstuffpb.SyncInfoFromProto(newViewFuzzMsg.SyncInfo) + newViewMsg.ID = hotstuff.ID(newViewFuzzMsg.ID) + return newViewMsg +} From 2a45bb4fc916276caf6eef54b234a5cfcaf5328a Mon Sep 17 00:00:00 2001 From: abje Date: Thu, 11 May 2023 14:32:07 +0200 Subject: [PATCH 03/25] initial commit, autogenerated protobuf file --- fuzz/fuzz.pb.go | 541 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 541 insertions(+) create mode 100644 fuzz/fuzz.pb.go diff --git a/fuzz/fuzz.pb.go b/fuzz/fuzz.pb.go new file mode 100644 index 000000000..959ae8cc8 --- /dev/null +++ b/fuzz/fuzz.pb.go @@ -0,0 +1,541 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.0 +// protoc v3.21.9 +// source: fuzz/fuzz.proto + +package fuzz + +import ( + hotstuffpb "github.com/relab/hotstuff/internal/proto/hotstuffpb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type FuzzMsg struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Message: + // + // *FuzzMsg_ProposeMsg + // *FuzzMsg_TimeoutMsg + // *FuzzMsg_VoteMsg + // *FuzzMsg_NewViewMsg + Message isFuzzMsg_Message `protobuf_oneof:"Message"` +} + +func (x *FuzzMsg) Reset() { + *x = FuzzMsg{} + if protoimpl.UnsafeEnabled { + mi := &file_fuzz_fuzz_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FuzzMsg) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FuzzMsg) ProtoMessage() {} + +func (x *FuzzMsg) ProtoReflect() protoreflect.Message { + mi := &file_fuzz_fuzz_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FuzzMsg.ProtoReflect.Descriptor instead. +func (*FuzzMsg) Descriptor() ([]byte, []int) { + return file_fuzz_fuzz_proto_rawDescGZIP(), []int{0} +} + +func (m *FuzzMsg) GetMessage() isFuzzMsg_Message { + if m != nil { + return m.Message + } + return nil +} + +func (x *FuzzMsg) GetProposeMsg() *ProposeMsg { + if x, ok := x.GetMessage().(*FuzzMsg_ProposeMsg); ok { + return x.ProposeMsg + } + return nil +} + +func (x *FuzzMsg) GetTimeoutMsg() *TimeoutMsg { + if x, ok := x.GetMessage().(*FuzzMsg_TimeoutMsg); ok { + return x.TimeoutMsg + } + return nil +} + +func (x *FuzzMsg) GetVoteMsg() *VoteMsg { + if x, ok := x.GetMessage().(*FuzzMsg_VoteMsg); ok { + return x.VoteMsg + } + return nil +} + +func (x *FuzzMsg) GetNewViewMsg() *NewViewMsg { + if x, ok := x.GetMessage().(*FuzzMsg_NewViewMsg); ok { + return x.NewViewMsg + } + return nil +} + +type isFuzzMsg_Message interface { + isFuzzMsg_Message() +} + +type FuzzMsg_ProposeMsg struct { + ProposeMsg *ProposeMsg `protobuf:"bytes,1,opt,name=ProposeMsg,proto3,oneof"` +} + +type FuzzMsg_TimeoutMsg struct { + TimeoutMsg *TimeoutMsg `protobuf:"bytes,2,opt,name=TimeoutMsg,proto3,oneof"` +} + +type FuzzMsg_VoteMsg struct { + VoteMsg *VoteMsg `protobuf:"bytes,3,opt,name=VoteMsg,proto3,oneof"` +} + +type FuzzMsg_NewViewMsg struct { + NewViewMsg *NewViewMsg `protobuf:"bytes,4,opt,name=NewViewMsg,proto3,oneof"` +} + +func (*FuzzMsg_ProposeMsg) isFuzzMsg_Message() {} + +func (*FuzzMsg_TimeoutMsg) isFuzzMsg_Message() {} + +func (*FuzzMsg_VoteMsg) isFuzzMsg_Message() {} + +func (*FuzzMsg_NewViewMsg) isFuzzMsg_Message() {} + +type ProposeMsg struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ID uint32 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + Proposal *hotstuffpb.Proposal `protobuf:"bytes,2,opt,name=Proposal,proto3" json:"Proposal,omitempty"` +} + +func (x *ProposeMsg) Reset() { + *x = ProposeMsg{} + if protoimpl.UnsafeEnabled { + mi := &file_fuzz_fuzz_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProposeMsg) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProposeMsg) ProtoMessage() {} + +func (x *ProposeMsg) ProtoReflect() protoreflect.Message { + mi := &file_fuzz_fuzz_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProposeMsg.ProtoReflect.Descriptor instead. +func (*ProposeMsg) Descriptor() ([]byte, []int) { + return file_fuzz_fuzz_proto_rawDescGZIP(), []int{1} +} + +func (x *ProposeMsg) GetID() uint32 { + if x != nil { + return x.ID + } + return 0 +} + +func (x *ProposeMsg) GetProposal() *hotstuffpb.Proposal { + if x != nil { + return x.Proposal + } + return nil +} + +type TimeoutMsg struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ID uint32 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + TimeoutMsg *hotstuffpb.TimeoutMsg `protobuf:"bytes,2,opt,name=TimeoutMsg,proto3" json:"TimeoutMsg,omitempty"` +} + +func (x *TimeoutMsg) Reset() { + *x = TimeoutMsg{} + if protoimpl.UnsafeEnabled { + mi := &file_fuzz_fuzz_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TimeoutMsg) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TimeoutMsg) ProtoMessage() {} + +func (x *TimeoutMsg) ProtoReflect() protoreflect.Message { + mi := &file_fuzz_fuzz_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TimeoutMsg.ProtoReflect.Descriptor instead. +func (*TimeoutMsg) Descriptor() ([]byte, []int) { + return file_fuzz_fuzz_proto_rawDescGZIP(), []int{2} +} + +func (x *TimeoutMsg) GetID() uint32 { + if x != nil { + return x.ID + } + return 0 +} + +func (x *TimeoutMsg) GetTimeoutMsg() *hotstuffpb.TimeoutMsg { + if x != nil { + return x.TimeoutMsg + } + return nil +} + +type VoteMsg struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ID uint32 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + PartialCert *hotstuffpb.PartialCert `protobuf:"bytes,2,opt,name=PartialCert,proto3" json:"PartialCert,omitempty"` + Deferred bool `protobuf:"varint,3,opt,name=Deferred,proto3" json:"Deferred,omitempty"` +} + +func (x *VoteMsg) Reset() { + *x = VoteMsg{} + if protoimpl.UnsafeEnabled { + mi := &file_fuzz_fuzz_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *VoteMsg) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VoteMsg) ProtoMessage() {} + +func (x *VoteMsg) ProtoReflect() protoreflect.Message { + mi := &file_fuzz_fuzz_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VoteMsg.ProtoReflect.Descriptor instead. +func (*VoteMsg) Descriptor() ([]byte, []int) { + return file_fuzz_fuzz_proto_rawDescGZIP(), []int{3} +} + +func (x *VoteMsg) GetID() uint32 { + if x != nil { + return x.ID + } + return 0 +} + +func (x *VoteMsg) GetPartialCert() *hotstuffpb.PartialCert { + if x != nil { + return x.PartialCert + } + return nil +} + +func (x *VoteMsg) GetDeferred() bool { + if x != nil { + return x.Deferred + } + return false +} + +type NewViewMsg struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ID uint32 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + SyncInfo *hotstuffpb.SyncInfo `protobuf:"bytes,2,opt,name=SyncInfo,proto3" json:"SyncInfo,omitempty"` +} + +func (x *NewViewMsg) Reset() { + *x = NewViewMsg{} + if protoimpl.UnsafeEnabled { + mi := &file_fuzz_fuzz_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NewViewMsg) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NewViewMsg) ProtoMessage() {} + +func (x *NewViewMsg) ProtoReflect() protoreflect.Message { + mi := &file_fuzz_fuzz_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NewViewMsg.ProtoReflect.Descriptor instead. +func (*NewViewMsg) Descriptor() ([]byte, []int) { + return file_fuzz_fuzz_proto_rawDescGZIP(), []int{4} +} + +func (x *NewViewMsg) GetID() uint32 { + if x != nil { + return x.ID + } + return 0 +} + +func (x *NewViewMsg) GetSyncInfo() *hotstuffpb.SyncInfo { + if x != nil { + return x.SyncInfo + } + return nil +} + +var File_fuzz_fuzz_proto protoreflect.FileDescriptor + +var file_fuzz_fuzz_proto_rawDesc = []byte{ + 0x0a, 0x0f, 0x66, 0x75, 0x7a, 0x7a, 0x2f, 0x66, 0x75, 0x7a, 0x7a, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x04, 0x66, 0x75, 0x7a, 0x7a, 0x1a, 0x28, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, + 0x70, 0x62, 0x2f, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x22, 0xdb, 0x01, 0x0a, 0x07, 0x46, 0x75, 0x7a, 0x7a, 0x4d, 0x73, 0x67, 0x12, 0x32, 0x0a, + 0x0a, 0x50, 0x72, 0x6f, 0x70, 0x6f, 0x73, 0x65, 0x4d, 0x73, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x10, 0x2e, 0x66, 0x75, 0x7a, 0x7a, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x6f, 0x73, 0x65, + 0x4d, 0x73, 0x67, 0x48, 0x00, 0x52, 0x0a, 0x50, 0x72, 0x6f, 0x70, 0x6f, 0x73, 0x65, 0x4d, 0x73, + 0x67, 0x12, 0x32, 0x0a, 0x0a, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x67, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x66, 0x75, 0x7a, 0x7a, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x67, 0x48, 0x00, 0x52, 0x0a, 0x54, 0x69, 0x6d, 0x65, 0x6f, + 0x75, 0x74, 0x4d, 0x73, 0x67, 0x12, 0x29, 0x0a, 0x07, 0x56, 0x6f, 0x74, 0x65, 0x4d, 0x73, 0x67, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x66, 0x75, 0x7a, 0x7a, 0x2e, 0x56, 0x6f, + 0x74, 0x65, 0x4d, 0x73, 0x67, 0x48, 0x00, 0x52, 0x07, 0x56, 0x6f, 0x74, 0x65, 0x4d, 0x73, 0x67, + 0x12, 0x32, 0x0a, 0x0a, 0x4e, 0x65, 0x77, 0x56, 0x69, 0x65, 0x77, 0x4d, 0x73, 0x67, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x66, 0x75, 0x7a, 0x7a, 0x2e, 0x4e, 0x65, 0x77, 0x56, + 0x69, 0x65, 0x77, 0x4d, 0x73, 0x67, 0x48, 0x00, 0x52, 0x0a, 0x4e, 0x65, 0x77, 0x56, 0x69, 0x65, + 0x77, 0x4d, 0x73, 0x67, 0x42, 0x09, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, + 0x4e, 0x0a, 0x0a, 0x50, 0x72, 0x6f, 0x70, 0x6f, 0x73, 0x65, 0x4d, 0x73, 0x67, 0x12, 0x0e, 0x0a, + 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x49, 0x44, 0x12, 0x30, 0x0a, + 0x08, 0x50, 0x72, 0x6f, 0x70, 0x6f, 0x73, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x14, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x50, 0x72, 0x6f, + 0x70, 0x6f, 0x73, 0x61, 0x6c, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x70, 0x6f, 0x73, 0x61, 0x6c, 0x22, + 0x54, 0x0a, 0x0a, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x67, 0x12, 0x0e, 0x0a, + 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x49, 0x44, 0x12, 0x36, 0x0a, + 0x0a, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x16, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x67, 0x52, 0x0a, 0x54, 0x69, 0x6d, 0x65, 0x6f, + 0x75, 0x74, 0x4d, 0x73, 0x67, 0x22, 0x70, 0x0a, 0x07, 0x56, 0x6f, 0x74, 0x65, 0x4d, 0x73, 0x67, + 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x49, 0x44, + 0x12, 0x39, 0x0a, 0x0b, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x65, 0x72, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, + 0x70, 0x62, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x65, 0x72, 0x74, 0x52, 0x0b, + 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x65, 0x72, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x44, + 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x44, + 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x22, 0x4e, 0x0a, 0x0a, 0x4e, 0x65, 0x77, 0x56, 0x69, + 0x65, 0x77, 0x4d, 0x73, 0x67, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x02, 0x49, 0x44, 0x12, 0x30, 0x0a, 0x08, 0x53, 0x79, 0x6e, 0x63, 0x49, 0x6e, 0x66, + 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, + 0x66, 0x66, 0x70, 0x62, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x53, + 0x79, 0x6e, 0x63, 0x49, 0x6e, 0x66, 0x6f, 0x42, 0x20, 0x5a, 0x1e, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x65, 0x6c, 0x61, 0x62, 0x2f, 0x68, 0x6f, 0x74, 0x73, + 0x74, 0x75, 0x66, 0x66, 0x2f, 0x66, 0x75, 0x7a, 0x7a, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, +} + +var ( + file_fuzz_fuzz_proto_rawDescOnce sync.Once + file_fuzz_fuzz_proto_rawDescData = file_fuzz_fuzz_proto_rawDesc +) + +func file_fuzz_fuzz_proto_rawDescGZIP() []byte { + file_fuzz_fuzz_proto_rawDescOnce.Do(func() { + file_fuzz_fuzz_proto_rawDescData = protoimpl.X.CompressGZIP(file_fuzz_fuzz_proto_rawDescData) + }) + return file_fuzz_fuzz_proto_rawDescData +} + +var file_fuzz_fuzz_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_fuzz_fuzz_proto_goTypes = []interface{}{ + (*FuzzMsg)(nil), // 0: fuzz.FuzzMsg + (*ProposeMsg)(nil), // 1: fuzz.ProposeMsg + (*TimeoutMsg)(nil), // 2: fuzz.TimeoutMsg + (*VoteMsg)(nil), // 3: fuzz.VoteMsg + (*NewViewMsg)(nil), // 4: fuzz.NewViewMsg + (*hotstuffpb.Proposal)(nil), // 5: hotstuffpb.Proposal + (*hotstuffpb.TimeoutMsg)(nil), // 6: hotstuffpb.TimeoutMsg + (*hotstuffpb.PartialCert)(nil), // 7: hotstuffpb.PartialCert + (*hotstuffpb.SyncInfo)(nil), // 8: hotstuffpb.SyncInfo +} +var file_fuzz_fuzz_proto_depIdxs = []int32{ + 1, // 0: fuzz.FuzzMsg.ProposeMsg:type_name -> fuzz.ProposeMsg + 2, // 1: fuzz.FuzzMsg.TimeoutMsg:type_name -> fuzz.TimeoutMsg + 3, // 2: fuzz.FuzzMsg.VoteMsg:type_name -> fuzz.VoteMsg + 4, // 3: fuzz.FuzzMsg.NewViewMsg:type_name -> fuzz.NewViewMsg + 5, // 4: fuzz.ProposeMsg.Proposal:type_name -> hotstuffpb.Proposal + 6, // 5: fuzz.TimeoutMsg.TimeoutMsg:type_name -> hotstuffpb.TimeoutMsg + 7, // 6: fuzz.VoteMsg.PartialCert:type_name -> hotstuffpb.PartialCert + 8, // 7: fuzz.NewViewMsg.SyncInfo:type_name -> hotstuffpb.SyncInfo + 8, // [8:8] is the sub-list for method output_type + 8, // [8:8] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name +} + +func init() { file_fuzz_fuzz_proto_init() } +func file_fuzz_fuzz_proto_init() { + if File_fuzz_fuzz_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_fuzz_fuzz_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FuzzMsg); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_fuzz_fuzz_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProposeMsg); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_fuzz_fuzz_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TimeoutMsg); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_fuzz_fuzz_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*VoteMsg); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_fuzz_fuzz_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NewViewMsg); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_fuzz_fuzz_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*FuzzMsg_ProposeMsg)(nil), + (*FuzzMsg_TimeoutMsg)(nil), + (*FuzzMsg_VoteMsg)(nil), + (*FuzzMsg_NewViewMsg)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_fuzz_fuzz_proto_rawDesc, + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_fuzz_fuzz_proto_goTypes, + DependencyIndexes: file_fuzz_fuzz_proto_depIdxs, + MessageInfos: file_fuzz_fuzz_proto_msgTypes, + }.Build() + File_fuzz_fuzz_proto = out.File + file_fuzz_fuzz_proto_rawDesc = nil + file_fuzz_fuzz_proto_goTypes = nil + file_fuzz_fuzz_proto_depIdxs = nil +} From ff49fa8d6701e5cf64b1c69031b0f0ef7fe437d5 Mon Sep 17 00:00:00 2001 From: abje Date: Thu, 11 May 2023 14:35:40 +0200 Subject: [PATCH 04/25] removing empty source file --- fuzz/convert.go | 1 - 1 file changed, 1 deletion(-) delete mode 100644 fuzz/convert.go diff --git a/fuzz/convert.go b/fuzz/convert.go deleted file mode 100644 index 504e8e0a0..000000000 --- a/fuzz/convert.go +++ /dev/null @@ -1 +0,0 @@ -package fuzz From 9fa9eeef1e03d13c8e60b949550c3c10dfb8e122 Mon Sep 17 00:00:00 2001 From: abje Date: Thu, 11 May 2023 14:36:21 +0200 Subject: [PATCH 05/25] added google/gofuzz as a dependency --- go.mod | 1 + go.sum | 1 + 2 files changed, 2 insertions(+) diff --git a/go.mod b/go.mod index 285baf9dd..363e34f25 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/felixge/fgprof v0.9.2 github.com/golang/mock v1.6.0 + github.com/google/gofuzz v1.1.0 github.com/kilic/bls12-381 v0.1.1-0.20210208205449-6045b0235e36 github.com/mattn/go-isatty v0.0.14 github.com/mitchellh/go-homedir v1.1.0 diff --git a/go.sum b/go.sum index 62edbc2ca..4bd2dcd1f 100644 --- a/go.sum +++ b/go.sum @@ -398,6 +398,7 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= From a05d0ec141847c56d2288d355056c14cb5083b9b Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Sat, 17 Jun 2023 11:12:32 -0700 Subject: [PATCH 06/25] Just formatting fixes --- fuzz/error.go | 8 +++----- fuzz/extra_test.go | 5 +---- fuzz/fuzz.go | 1 - fuzz/fuzz_test.go | 4 ---- fuzz/network.go | 2 -- fuzz/serialize.go | 5 ----- fuzz/tostring.go | 16 ---------------- 7 files changed, 4 insertions(+), 37 deletions(-) diff --git a/fuzz/error.go b/fuzz/error.go index 24da7345a..5dca73885 100644 --- a/fuzz/error.go +++ b/fuzz/error.go @@ -2,7 +2,7 @@ package fuzz import ( "fmt" - reflect "reflect" + "reflect" "sort" "strconv" "strings" @@ -62,7 +62,6 @@ func (errorInfo *ErrorInfo) Init() { } func (errorInfo *ErrorInfo) OutputInfo(t *testing.T) { - b64s := "" seeds := "" @@ -77,7 +76,7 @@ func (errorInfo *ErrorInfo) OutputInfo(t *testing.T) { keys = append(keys, key) } - //sorting the keys of the + // sorting the keys of the sort.Strings(keys) for i, key := range keys { @@ -90,7 +89,7 @@ func (errorInfo *ErrorInfo) OutputInfo(t *testing.T) { fmt.Println() fmt.Printf("ERROR NUMBER %d\n", i+1) - //contains error location, err text and recover point + // contains error location, err text and recover point fmt.Println(key) fmt.Println() fmt.Println("crash amounts grouped by type:") @@ -135,7 +134,6 @@ func (errorInfo *ErrorInfo) AddTotal(fuzzMessage *FuzzMsg, seed *int64) { } func (errorInfo *ErrorInfo) AddPanic(fullStack string, err2 any, info string) { - simpleStack := SimplifyStack(fullStack) identifier := "error location:\t" + simpleStack + "\nerror info:\t" + fmt.Sprint(err2) + "\nrecovered from:\t" + info diff --git a/fuzz/extra_test.go b/fuzz/extra_test.go index dd5687452..3a273ce11 100644 --- a/fuzz/extra_test.go +++ b/fuzz/extra_test.go @@ -12,7 +12,6 @@ import ( ) func TestFrequencyErrorFuzz(t *testing.T) { - frequency := make(map[string]int, 0) f := initFuzz() @@ -42,7 +41,6 @@ func TestFrequencyErrorFuzz(t *testing.T) { } fmt.Println(sum) - } func BenchmarkFuzz(b *testing.B) { @@ -59,13 +57,12 @@ func BenchmarkFuzz(b *testing.B) { fmt.Println(b.Elapsed()) - //errorInfo.OutputInfo(nil) + // errorInfo.OutputInfo(nil) fmt.Println() } func TestExperimentalString(t *testing.T) { - f := initFuzz() fuzzMessage := createFuzzMessage(f, nil) diff --git a/fuzz/fuzz.go b/fuzz/fuzz.go index 452430b8d..0d3dcd7ba 100644 --- a/fuzz/fuzz.go +++ b/fuzz/fuzz.go @@ -63,7 +63,6 @@ func initFuzz() *fuzz.Fuzzer { } func createFuzzMessage(f *fuzz.Fuzzer, seed *int64) *FuzzMsg { - if seed != nil { f.RandSource(rand.NewSource(*seed)) } diff --git a/fuzz/fuzz_test.go b/fuzz/fuzz_test.go index 97d66a628..5dccfa559 100644 --- a/fuzz/fuzz_test.go +++ b/fuzz/fuzz_test.go @@ -34,7 +34,6 @@ func TryExecuteScenario(errorInfo *ErrorInfo, oldMessage any, newMessage any) { s = append(s, View{Leader: 1, Partitions: []NodeSet{allNodesSet}}) result, err := ExecuteScenario(s, numNodes, 0, 100, "chainedhotstuff", oldMessage, newMessage) - if err != nil { panic(err) } @@ -122,12 +121,10 @@ func TestFuzz(t *testing.T) { // load previously created fuzz messages from a file // it doesn't work quite right, i blame proto.Marshal() func TestPreviousFuzz(t *testing.T) { - errorInfo := new(ErrorInfo) errorInfo.Init() fuzzMsgs, err := loadFuzzMessagesFromFile("previous_messages.b64") - if err != nil { panic(err) } @@ -146,7 +143,6 @@ func TestSeedPreviousFuzz(t *testing.T) { errorInfo.Init() seeds, err := loadSeedsFromFile("previous_messages.seed") - if err != nil { panic(err) } diff --git a/fuzz/network.go b/fuzz/network.go index 5f6433f3a..8e1426fd5 100644 --- a/fuzz/network.go +++ b/fuzz/network.go @@ -195,7 +195,6 @@ func (n *Network) tick() { // shouldDrop decides if the sender should drop the message, based on the current view of the sender and the // partitions configured for that view. func (n *Network) shouldDrop(sender, receiver uint32, message any) bool { - node, ok := n.nodes[sender] if !ok { panic(fmt.Errorf("node matching sender id %d was not found", sender)) @@ -266,7 +265,6 @@ func (c *configuration) broadcastMessage(message any) { } func (c *configuration) sendMessage(id hotstuff.ID, message any) { - nodes, ok := c.network.replicas[id] if !ok { panic(fmt.Errorf("attempt to send message to replica %d, but this replica does not exist", id)) diff --git a/fuzz/serialize.go b/fuzz/serialize.go index 0129adcde..0a24d8fe1 100644 --- a/fuzz/serialize.go +++ b/fuzz/serialize.go @@ -40,7 +40,6 @@ func b64ToFuzzMsg(str string) (*FuzzMsg, error) { }*/ func saveStringToFile(filename string, str string) error { - f, err := os.Create(filename) if err != nil { return err @@ -74,9 +73,7 @@ func loadStringFromFile(filename string) (string, error) { } func loadFuzzMessagesFromFile(filename string) ([]*FuzzMsg, error) { - str, err := loadStringFromFile(filename) - if err != nil { return nil, err } @@ -102,9 +99,7 @@ func loadFuzzMessagesFromFile(filename string) ([]*FuzzMsg, error) { } func loadSeedsFromFile(filename string) ([]int64, error) { - str, err := loadStringFromFile(filename) - if err != nil { return nil, err } diff --git a/fuzz/tostring.go b/fuzz/tostring.go index 58a7c4adc..1f03b489c 100644 --- a/fuzz/tostring.go +++ b/fuzz/tostring.go @@ -27,7 +27,6 @@ func (proposeFuzzMsg *ProposeMsg) ToString(depth int) string { } func (timeoutFuzzMsg *TimeoutMsg) ToString(depth int) string { - tabs := depthToTabs(depth) return fmt.Sprintf( @@ -56,7 +55,6 @@ func (voteFuzzMsg *VoteMsg) ToString(depth int) string { } func (newViewFuzzMsg *NewViewMsg) ToString(depth int) string { - if newViewFuzzMsg == nil { return "nil" } @@ -74,7 +72,6 @@ func (newViewFuzzMsg *NewViewMsg) ToString(depth int) string { } func ProposalToString(proposal *hotstuffpb.Proposal, depth int) string { - if proposal == nil { return "nil" } @@ -92,7 +89,6 @@ func ProposalToString(proposal *hotstuffpb.Proposal, depth int) string { } func BlockToString(block *hotstuffpb.Block, depth int) string { - if block == nil { return "nil" } @@ -116,7 +112,6 @@ func BlockToString(block *hotstuffpb.Block, depth int) string { } func TimeoutMsgToString(timeoutMsg *hotstuffpb.TimeoutMsg, depth int) string { - if timeoutMsg == nil { return "nil" } @@ -146,7 +141,6 @@ func SyncInfoPtrToString(object *hotstuffpb.SyncInfo, depth int) string { } func SyncInfoToString(syncInfo *hotstuffpb.SyncInfo, depth int) string { - if syncInfo == nil { return "nil" } @@ -166,7 +160,6 @@ func SyncInfoToString(syncInfo *hotstuffpb.SyncInfo, depth int) string { } func QuorumCertToString(qc *hotstuffpb.QuorumCert, depth int) string { - if qc == nil { return "nil" } @@ -186,7 +179,6 @@ func QuorumCertToString(qc *hotstuffpb.QuorumCert, depth int) string { } func TimeoutCertToString(tc *hotstuffpb.TimeoutCert, depth int) string { - if tc == nil { return "nil" } @@ -204,7 +196,6 @@ func TimeoutCertToString(tc *hotstuffpb.TimeoutCert, depth int) string { } func AggQCToString(aggQC *hotstuffpb.AggQC, depth int) string { - if aggQC == nil { return "nil" } @@ -232,7 +223,6 @@ func AggQCToString(aggQC *hotstuffpb.AggQC, depth int) string { } func PartialCertToString(partialCert *hotstuffpb.PartialCert, depth int) string { - if partialCert == nil { return "nil" } @@ -250,7 +240,6 @@ func PartialCertToString(partialCert *hotstuffpb.PartialCert, depth int) string } func QuorumSignatureToString(quorumSignature *hotstuffpb.QuorumSignature, depth int) string { - if quorumSignature == nil { return "nil" } @@ -278,7 +267,6 @@ func QuorumSignatureToString(quorumSignature *hotstuffpb.QuorumSignature, depth } func QuorumSignature_ECDSASigsToString(ECDASigs *hotstuffpb.QuorumSignature_ECDSASigs, depth int) string { - if ECDASigs == nil { return "nil" } @@ -294,7 +282,6 @@ func QuorumSignature_ECDSASigsToString(ECDASigs *hotstuffpb.QuorumSignature_ECDS } func ECDSAMultiSignatureToString(ECDASigs *hotstuffpb.ECDSAMultiSignature, depth int) string { - if ECDASigs == nil { return "nil" } @@ -318,7 +305,6 @@ func ECDSAMultiSignatureToString(ECDASigs *hotstuffpb.ECDSAMultiSignature, depth } func ECDASigToString(sig *hotstuffpb.ECDSASignature, depth int) string { - if sig == nil { return "nil" } @@ -338,7 +324,6 @@ func ECDASigToString(sig *hotstuffpb.ECDSASignature, depth int) string { } func QuorumSignature_BLS12SigToString(BLS12Sig *hotstuffpb.QuorumSignature_BLS12Sig, depth int) string { - if BLS12Sig == nil { return "nil" } @@ -354,7 +339,6 @@ func QuorumSignature_BLS12SigToString(BLS12Sig *hotstuffpb.QuorumSignature_BLS12 } func BLS12AggregateSignatureToString(BLS12Sig *hotstuffpb.BLS12AggregateSignature, depth int) string { - if BLS12Sig == nil { return "nil" } From 5dc5bb2758449a063bf775bb24f96a864febe16f Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Sat, 17 Jun 2023 12:47:48 -0700 Subject: [PATCH 07/25] Formatting fuzz.proto file --- fuzz/fuzz.proto | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/fuzz/fuzz.proto b/fuzz/fuzz.proto index f87efbf61..58127d188 100644 --- a/fuzz/fuzz.proto +++ b/fuzz/fuzz.proto @@ -7,31 +7,31 @@ import "internal/proto/hotstuffpb/hotstuff.proto"; option go_package = "github.com/relab/hotstuff/fuzz"; message FuzzMsg { - oneof Message { - ProposeMsg ProposeMsg = 1; - TimeoutMsg TimeoutMsg = 2; - VoteMsg VoteMsg = 3; - NewViewMsg NewViewMsg = 4; - } + oneof Message { + ProposeMsg ProposeMsg = 1; + TimeoutMsg TimeoutMsg = 2; + VoteMsg VoteMsg = 3; + NewViewMsg NewViewMsg = 4; + } } message ProposeMsg { - uint32 ID = 1; - hotstuffpb.Proposal Proposal = 2; + uint32 ID = 1; + hotstuffpb.Proposal Proposal = 2; } message TimeoutMsg { - uint32 ID = 1; - hotstuffpb.TimeoutMsg TimeoutMsg = 2; + uint32 ID = 1; + hotstuffpb.TimeoutMsg TimeoutMsg = 2; } message VoteMsg { - uint32 ID = 1; - hotstuffpb.PartialCert PartialCert = 2; - bool Deferred = 3; + uint32 ID = 1; + hotstuffpb.PartialCert PartialCert = 2; + bool Deferred = 3; } message NewViewMsg { - uint32 ID = 1; - hotstuffpb.SyncInfo SyncInfo = 2; -} \ No newline at end of file + uint32 ID = 1; + hotstuffpb.SyncInfo SyncInfo = 2; +} From 644a9935393a07f409d04957902a4240e734d22f Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Sat, 17 Jun 2023 13:02:33 -0700 Subject: [PATCH 08/25] Fixed a few lint minor lint issues --- .vscode/dict.txt | 3 +++ fuzz/experimental_string.go | 6 +----- fuzz/network.go | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.vscode/dict.txt b/.vscode/dict.txt index 58defd596..d99530512 100644 --- a/.vscode/dict.txt +++ b/.vscode/dict.txt @@ -48,6 +48,8 @@ Mathieu Meling mitchellh nolint +Oneof +Oneofs orchestrationpb partitioner Paulo @@ -59,6 +61,7 @@ propsed proto protobuf protoc +protoreflect protostream ptypes QC's diff --git a/fuzz/experimental_string.go b/fuzz/experimental_string.go index f946a34d9..6ce684747 100644 --- a/fuzz/experimental_string.go +++ b/fuzz/experimental_string.go @@ -16,13 +16,11 @@ func ValueToString(v protoreflect.Value, depth int) string { if ok { return ProtoToString(v.Message(), depth+1) - } else { - return InterfaceToString(v.Interface()) } + return InterfaceToString(v.Interface()) } func ListToString(list protoreflect.List, depth int) string { - if list.Len() == 0 { return "[]" } @@ -42,7 +40,6 @@ func ListToString(list protoreflect.List, depth int) string { } func MapToString(mp protoreflect.Map, depth int) string { - if mp.Len() == 0 { return "[]" } @@ -98,7 +95,6 @@ func FieldToString(field protoreflect.FieldDescriptor, prm protoreflect.Message, } func ProtoToString(prm protoreflect.Message, depth int) string { - if !prm.IsValid() { return "nil\n" } diff --git a/fuzz/network.go b/fuzz/network.go index 8e1426fd5..27f08d84a 100644 --- a/fuzz/network.go +++ b/fuzz/network.go @@ -278,7 +278,7 @@ func (c *configuration) sendMessage(id hotstuff.ID, message any) { } c.network.Messages = append(c.network.Messages, message) - c.network.MessageCounter += 1 + c.network.MessageCounter++ if c.network.shouldSwap(message) { From 07d6d1c42f8dab89c2012204fbd3ae12d59573da Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Sat, 17 Jun 2023 13:03:06 -0700 Subject: [PATCH 09/25] Fixed typos --- .vscode/dict.txt | 3 +++ fuzz/extra_test.go | 2 +- fuzz/tostring.go | 24 ++++++++++++------------ 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.vscode/dict.txt b/.vscode/dict.txt index d99530512..528e37382 100644 --- a/.vscode/dict.txt +++ b/.vscode/dict.txt @@ -16,6 +16,7 @@ Debugf durationpb emptypb Erevik +eventloop Fangyu fasthotstuff felixge @@ -41,6 +42,7 @@ Infof Jalalzai Jehl Jianyu +keygen kilic latencygen Malkhi @@ -74,6 +76,7 @@ simplehotstuff Sonnino SSWU structs +subconfiguration throughputvslatency timestamppb TMPDIR diff --git a/fuzz/extra_test.go b/fuzz/extra_test.go index 3a273ce11..356ae32c1 100644 --- a/fuzz/extra_test.go +++ b/fuzz/extra_test.go @@ -1,7 +1,7 @@ package fuzz // this file includes a couple of extra unnecessary tests -// used for propability tests, profiling and other things +// used for probability tests, profiling and other things import ( "fmt" diff --git a/fuzz/tostring.go b/fuzz/tostring.go index 1f03b489c..32bfba01d 100644 --- a/fuzz/tostring.go +++ b/fuzz/tostring.go @@ -45,7 +45,7 @@ func (voteFuzzMsg *VoteMsg) ToString(depth int) string { return fmt.Sprintf( "fuzz.VoteMsg{\n"+ "%s\tID: %v\n"+ - "%s\tDeffered: %v\n"+ + "%s\tDeferred: %v\n"+ "%s\tPartialCert: %v\n"+ "%s}", tabs, voteFuzzMsg.ID, @@ -266,8 +266,8 @@ func QuorumSignatureToString(quorumSignature *hotstuffpb.QuorumSignature, depth tabs) } -func QuorumSignature_ECDSASigsToString(ECDASigs *hotstuffpb.QuorumSignature_ECDSASigs, depth int) string { - if ECDASigs == nil { +func QuorumSignature_ECDSASigsToString(ECDSASigs *hotstuffpb.QuorumSignature_ECDSASigs, depth int) string { + if ECDSASigs == nil { return "nil" } @@ -275,36 +275,36 @@ func QuorumSignature_ECDSASigsToString(ECDASigs *hotstuffpb.QuorumSignature_ECDS return fmt.Sprintf( "&hotstuffpb.QuorumSignature_ECDSASigs{\n"+ - "%s\tECDASigs: %s\n"+ + "%s\tECDSASigs: %s\n"+ "%s},", - tabs, ECDSAMultiSignatureToString(ECDASigs.ECDSASigs, depth+1), + tabs, ECDSAMultiSignatureToString(ECDSASigs.ECDSASigs, depth+1), tabs) } -func ECDSAMultiSignatureToString(ECDASigs *hotstuffpb.ECDSAMultiSignature, depth int) string { - if ECDASigs == nil { +func ECDSAMultiSignatureToString(ECDSASigs *hotstuffpb.ECDSAMultiSignature, depth int) string { + if ECDSASigs == nil { return "nil" } tabs := depthToTabs(depth) - sigsString := "[]*ECDASignature{\n" + sigsString := "[]*ECDSASignature{\n" - for _, sig := range ECDASigs.Sigs { - sigsString += fmt.Sprintf("%s\t\t%s\n", tabs, ECDASigToString(sig, depth+2)) + for _, sig := range ECDSASigs.Sigs { + sigsString += fmt.Sprintf("%s\t\t%s\n", tabs, ECDSASigToString(sig, depth+2)) } sigsString += tabs + "\t}" return fmt.Sprintf( "&hotstuffpb.ECDSAMultiSignature{\n"+ - "%s\tECDASigs: %s\n"+ + "%s\tECDSASigs: %s\n"+ "%s}", tabs, sigsString, tabs) } -func ECDASigToString(sig *hotstuffpb.ECDSASignature, depth int) string { +func ECDSASigToString(sig *hotstuffpb.ECDSASignature, depth int) string { if sig == nil { return "nil" } From 8d9c23b23371ba95c90efa3ebdd2e1ca982724df Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Sat, 17 Jun 2023 13:19:51 -0700 Subject: [PATCH 10/25] Ignore fuzz data --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 8e5b49643..d62880d28 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,5 @@ measurements.json *.pdf twins.json +fuzz/previous_messages.b64 +fuzz/previous_messages.seed From 122f7e1e0e3c607b16e3acb48a6b03cbb234d11a Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Sat, 17 Jun 2023 13:27:18 -0700 Subject: [PATCH 11/25] Unexported experimental_string.go funcs --- fuzz/experimental_string.go | 35 +++++++++++++++-------------------- fuzz/extra_test.go | 2 +- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/fuzz/experimental_string.go b/fuzz/experimental_string.go index 6ce684747..87261da19 100644 --- a/fuzz/experimental_string.go +++ b/fuzz/experimental_string.go @@ -7,20 +7,19 @@ import ( ) // work in progress -func InterfaceToString(i interface{}) string { +func interfaceToString(i interface{}) string { return fmt.Sprintf("%v", i) } -func ValueToString(v protoreflect.Value, depth int) string { +func valueToString(v protoreflect.Value, depth int) string { _, ok := v.Interface().(protoreflect.Message) - if ok { - return ProtoToString(v.Message(), depth+1) + return protoToString(v.Message(), depth+1) } - return InterfaceToString(v.Interface()) + return interfaceToString(v.Interface()) } -func ListToString(list protoreflect.List, depth int) string { +func listToString(list protoreflect.List, depth int) string { if list.Len() == 0 { return "[]" } @@ -31,7 +30,7 @@ func ListToString(list protoreflect.List, depth int) string { for i := 0; i < list.Len(); i++ { item := list.Get(i) - str += tabs + "\t" + ValueToString(item, depth) + ",\n" + str += tabs + "\t" + valueToString(item, depth) + ",\n" } str += tabs + "]" @@ -39,7 +38,7 @@ func ListToString(list protoreflect.List, depth int) string { return str } -func MapToString(mp protoreflect.Map, depth int) string { +func mapToString(mp protoreflect.Map, depth int) string { if mp.Len() == 0 { return "[]" } @@ -48,17 +47,15 @@ func MapToString(mp protoreflect.Map, depth int) string { str := "[\n" mp.Range(func(mk protoreflect.MapKey, v protoreflect.Value) bool { - str += tabs + "\t" + ValueToString(mk.Value(), depth) + ": " + ValueToString(v, depth) + ",\n" - + str += tabs + "\t" + valueToString(mk.Value(), depth) + ": " + valueToString(v, depth) + ",\n" return true }) str += tabs + "]" - return str } -func FieldToString(field protoreflect.FieldDescriptor, prm protoreflect.Message, depth int) string { +func fieldToString(field protoreflect.FieldDescriptor, prm protoreflect.Message, depth int) string { msg := field.Message() str := "" @@ -69,7 +66,7 @@ func FieldToString(field protoreflect.FieldDescriptor, prm protoreflect.Message, if field.IsList() { list := prm.Get(field).List() - return str + ListToString(list, depth) + "\n" + return str + listToString(list, depth) + "\n" } if field.IsExtension() { @@ -78,7 +75,7 @@ func FieldToString(field protoreflect.FieldDescriptor, prm protoreflect.Message, if field.IsMap() { map2 := prm.Get(field).Map() - return str + MapToString(map2, depth) + "\n" + return str + mapToString(map2, depth) + "\n" } if field.IsPlaceholder() { @@ -86,15 +83,13 @@ func FieldToString(field protoreflect.FieldDescriptor, prm protoreflect.Message, } refl := prm.Get(field).Message() - if refl == nil { panic("message reflection is nil") } - - return str + ProtoToString(refl, depth) + "\n" + return str + protoToString(refl, depth) + "\n" } -func ProtoToString(prm protoreflect.Message, depth int) string { +func protoToString(prm protoreflect.Message, depth int) string { if !prm.IsValid() { return "nil\n" } @@ -111,7 +106,7 @@ func ProtoToString(prm protoreflect.Message, depth int) string { continue } - str += tabs + "\t" + string(field.Name()) + ": " + FieldToString(field, prm, depth+1) + str += tabs + "\t" + string(field.Name()) + ": " + fieldToString(field, prm, depth+1) } oneofs := desc.Oneofs() @@ -129,7 +124,7 @@ func ProtoToString(prm protoreflect.Message, depth int) string { continue } - str += tabs + "\t" + string(oneof.Name()) + ": " + FieldToString(field, prm, depth+1) + str += tabs + "\t" + string(oneof.Name()) + ": " + fieldToString(field, prm, depth+1) } } diff --git a/fuzz/extra_test.go b/fuzz/extra_test.go index 356ae32c1..aeeca4267 100644 --- a/fuzz/extra_test.go +++ b/fuzz/extra_test.go @@ -75,5 +75,5 @@ func TestExperimentalString(t *testing.T) { var msg proto.Message msg = fuzzMessage - fmt.Println(ProtoToString(msg.ProtoReflect(), 0)) + fmt.Println(protoToString(msg.ProtoReflect(), 0)) } From 33dadc6aa938946fa92a9d42b566c1a2d5218f1c Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Thu, 29 Jun 2023 15:38:15 -0700 Subject: [PATCH 12/25] Clean up the proto to Go struct literal and more This commit replaces the tostring.go file with the proto2.GoString() function that takes a generic proto.Message and returns a Go struct literal. This commit also replaces the interfaces AlmostFuzzMsg and FuzzMsgInterface with two simple helpers extractProtoMsg() and fuzzMsgToHotStuffMsg(). These are easier to use and IMO easier to maintain. Additionally, there are a few smaller tweaks. --- .vscode/extensions.json | 10 + .vscode/settings.json | 17 +- fuzz/error.go | 14 +- fuzz/experimental_string.go | 7 + fuzz/fuzz_test.go | 5 +- fuzz/tostring.go | 356 ------------------------------------ fuzz/types.go | 103 +++++------ go.mod | 5 +- go.sum | 9 +- 9 files changed, 100 insertions(+), 426 deletions(-) create mode 100644 .vscode/extensions.json delete mode 100644 fuzz/tostring.go diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 000000000..c27bc02e3 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,10 @@ +{ + "recommendations": [ + "xaver.clang-format", + "streetsidesoftware.code-spell-checker", + "golang.go", + "davidanson.vscode-markdownlint", + "zxh404.vscode-proto3", + "github.vscode-pull-request-github", + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 249e8d259..4ea01e578 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,21 @@ { + "clang-format.style": "{ IndentWidth: 4, BasedOnStyle: google, AlignConsecutiveAssignments: true, ColumnLimit: 120 }", + "editor.formatOnSave": true, + "protoc": { + "compile_on_save": false, + "options": [ + "--proto_path=${workspaceRoot}/", + "--proto_path=/Users/meling/go/pkg/mod/github.com/relab/gorums@v0.7.1-0.20220307181651-94a8af8e467c", + // "--proto_path=$(shell go list -m -f {{.Dir}} github.com/relab/gorums)", + // "--proto_path=`go list -m -f {{.Dir}} github.com/relab/gorums`", + ] + }, + "go.lintTool": "golangci-lint", + "go.lintFlags": [ + "--fast", + ], "go.testFlags": [ "-count=1" ], "cSpell.enabled": true -} +} \ No newline at end of file diff --git a/fuzz/error.go b/fuzz/error.go index 5dca73885..76ec1175a 100644 --- a/fuzz/error.go +++ b/fuzz/error.go @@ -7,6 +7,8 @@ import ( "strconv" "strings" "testing" + + "github.com/meling/proto2" ) type TypeCount map[reflect.Type]int @@ -129,7 +131,8 @@ func (errorInfo *ErrorInfo) AddTotal(fuzzMessage *FuzzMsg, seed *int64) { errorInfo.totalMessages++ errorInfo.currentFuzzMsg = fuzzMessage errorInfo.currentFuzzMsgSeed = seed - typ := reflect.TypeOf(fuzzMessage.Msg()) + protoMsg := extractProtoMsg(errorInfo.currentFuzzMsg) + typ := reflect.TypeOf(protoMsg) errorInfo.TypeTotalCount.Add(typ) } @@ -146,13 +149,14 @@ func (errorInfo *ErrorInfo) AddPanic(fullStack string, err2 any, info string) { panic(err) } - FuzzMsgString := errorInfo.currentFuzzMsg.Msg().ToString(0) - newLines := strings.Count(FuzzMsgString, "\n") + protoMsg := extractProtoMsg(errorInfo.currentFuzzMsg) + fuzzMsgString := proto2.GoString(protoMsg) + newLines := strings.Count(fuzzMsgString, "\n") newPanic := PanicInfo{ Err: err2, StackTrace: fullStack, - FuzzMsg: FuzzMsgString, + FuzzMsg: fuzzMsgString, FuzzMsgB64: b64, Seed: errorInfo.currentFuzzMsgSeed, LineNum: newLines, @@ -168,7 +172,7 @@ func (errorInfo *ErrorInfo) AddPanic(fullStack string, err2 any, info string) { if !okPanic || newLines < oldLines { errorInfo.panics[identifier] = newPanic } - typ := reflect.TypeOf(errorInfo.currentFuzzMsg.Msg()) + typ := reflect.TypeOf(protoMsg) errorInfo.panics[identifier].TypeCount.Add(typ) errorInfo.TypePanicCount.Add(typ) } diff --git a/fuzz/experimental_string.go b/fuzz/experimental_string.go index 87261da19..e4aa20fde 100644 --- a/fuzz/experimental_string.go +++ b/fuzz/experimental_string.go @@ -6,6 +6,13 @@ import ( "google.golang.org/protobuf/reflect/protoreflect" ) +func depthToTabs(depth int) (tabs string) { + for i := 0; i < depth; i++ { + tabs += "\t" + } + return +} + // work in progress func interfaceToString(i interface{}) string { return fmt.Sprintf("%v", i) diff --git a/fuzz/fuzz_test.go b/fuzz/fuzz_test.go index 5dccfa559..1adb9bedc 100644 --- a/fuzz/fuzz_test.go +++ b/fuzz/fuzz_test.go @@ -82,15 +82,12 @@ func fuzzMsgToMsg(errorInfo *ErrorInfo, fuzzMsg *FuzzMsg) any { errorInfo.failedMessages++ } }() - - return fuzzMsg.Msg().ToMsg() + return fuzzMsgToHotStuffMsg(fuzzMsg) } func useFuzzMessage(errorInfo *ErrorInfo, fuzzMessage *FuzzMsg, seed *int64) { errorInfo.AddTotal(fuzzMessage, seed) - newMessage := fuzzMsgToMsg(errorInfo, fuzzMessage) - if newMessage != nil { fuzzScenario(errorInfo, newMessage) } diff --git a/fuzz/tostring.go b/fuzz/tostring.go deleted file mode 100644 index 32bfba01d..000000000 --- a/fuzz/tostring.go +++ /dev/null @@ -1,356 +0,0 @@ -package fuzz - -import ( - "fmt" - - "github.com/relab/hotstuff/internal/proto/hotstuffpb" -) - -func depthToTabs(depth int) (tabs string) { - for i := 0; i < depth; i++ { - tabs += "\t" - } - return -} - -func (proposeFuzzMsg *ProposeMsg) ToString(depth int) string { - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "fuzz.ProposeMsg{\n"+ - "%s\tID: %v\n"+ - "%s\tProposal: %v\n"+ - "%s}", - tabs, proposeFuzzMsg.ID, - tabs, ProposalToString(proposeFuzzMsg.Proposal, depth+1), - tabs) -} - -func (timeoutFuzzMsg *TimeoutMsg) ToString(depth int) string { - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "fuzz.TimeoutMsg{\n"+ - "%s\tID: %v\n"+ - "%s\tTimeoutMsg: %v\n"+ - "%s}", - tabs, timeoutFuzzMsg.ID, - tabs, TimeoutMsgToString(timeoutFuzzMsg.TimeoutMsg, depth+1), - tabs) -} - -func (voteFuzzMsg *VoteMsg) ToString(depth int) string { - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "fuzz.VoteMsg{\n"+ - "%s\tID: %v\n"+ - "%s\tDeferred: %v\n"+ - "%s\tPartialCert: %v\n"+ - "%s}", - tabs, voteFuzzMsg.ID, - tabs, voteFuzzMsg.Deferred, - tabs, PartialCertToString(voteFuzzMsg.PartialCert, depth+1), - tabs) -} - -func (newViewFuzzMsg *NewViewMsg) ToString(depth int) string { - if newViewFuzzMsg == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "fuzz.NewViewMsg{\n"+ - "%s\tID: %v\n"+ - "%s\tSyncInfo: %v\n"+ - "%s}", - tabs, newViewFuzzMsg.ID, - tabs, SyncInfoToString(newViewFuzzMsg.SyncInfo, depth+1), - tabs) -} - -func ProposalToString(proposal *hotstuffpb.Proposal, depth int) string { - if proposal == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "hotstuffpb.Proposal{\n"+ - "%s\tBlock: %v\n"+ - "%s\tAggQC: %v\n"+ - "%s}", - tabs, BlockToString(proposal.Block, depth+1), - tabs, AggQCToString(proposal.AggQC, depth+1), - tabs) -} - -func BlockToString(block *hotstuffpb.Block, depth int) string { - if block == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "&hotstuffpb.Block{\n"+ - "%s\tParent: %v\n"+ - "%s\tQC: %v\n"+ - "%s\tView: %v\n"+ - "%s\tCommand: %v\n"+ - "%s\tProposer: %v\n"+ - "%s}", - tabs, block.Parent, - tabs, QuorumCertToString(block.QC, depth+1), - tabs, block.View, - tabs, block.Command, - tabs, block.Proposer, - tabs) -} - -func TimeoutMsgToString(timeoutMsg *hotstuffpb.TimeoutMsg, depth int) string { - if timeoutMsg == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "hotstuffpb.TimeoutMsg{\n"+ - "%s\tView: %v\n"+ - "%s\tSyncInfo: %v\n"+ - "%s\tViewSig: %v\n"+ - "%s\tMsgSig: %v\n"+ - "%s}", - tabs, timeoutMsg.View, - tabs, SyncInfoPtrToString(timeoutMsg.SyncInfo, depth+1), - tabs, QuorumSignatureToString(timeoutMsg.ViewSig, depth+1), - tabs, QuorumSignatureToString(timeoutMsg.MsgSig, depth+1), - tabs) -} - -func SyncInfoPtrToString(object *hotstuffpb.SyncInfo, depth int) string { - if object == nil { - return "nil" - } - - return "&" + SyncInfoToString(object, depth) -} - -func SyncInfoToString(syncInfo *hotstuffpb.SyncInfo, depth int) string { - if syncInfo == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "hotstuffpb.SyncInfo{\n"+ - "%s\tQC: %v\n"+ - "%s\tTC: %v\n"+ - "%s\tAggQC: %v\n"+ - "%s}", - tabs, QuorumCertToString(syncInfo.QC, depth+1), - tabs, TimeoutCertToString(syncInfo.TC, depth+1), - tabs, AggQCToString(syncInfo.AggQC, depth+1), - tabs) -} - -func QuorumCertToString(qc *hotstuffpb.QuorumCert, depth int) string { - if qc == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "&hotstuffpb.QuorumCert{\n"+ - "%s\tSig: %v\n"+ - "%s\tView: %v\n"+ - "%s\tHash: %v\n"+ - "%s}", - tabs, QuorumSignatureToString(qc.Sig, depth+1), - tabs, qc.View, - tabs, qc.Hash, - tabs) -} - -func TimeoutCertToString(tc *hotstuffpb.TimeoutCert, depth int) string { - if tc == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "&hotstuffpb.TimeoutCert{\n"+ - "%s\tSig: %v\n"+ - "%s\tView: %v\n"+ - "%s}", - tabs, QuorumSignatureToString(tc.Sig, depth+1), - tabs, tc.View, - tabs) -} - -func AggQCToString(aggQC *hotstuffpb.AggQC, depth int) string { - if aggQC == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - QCsString := "map[uint32]*hotstuffpb.QuorumCert{\n" - - for key, value := range aggQC.QCs { - QCsString += fmt.Sprintf("%v\t\t%v: %v\n", tabs, key, QuorumCertToString(value, depth+2)) - } - - QCsString += tabs + "\t" - - return fmt.Sprintf( - "&hotstuffpb.AggQC{\n"+ - "%s\tQCs: %v\n"+ - "%s\tSig: %v\n"+ - "%s\tView: %v\n"+ - "%s}", - tabs, QCsString, - tabs, QuorumSignatureToString(aggQC.Sig, depth+1), - tabs, aggQC.View, - tabs) -} - -func PartialCertToString(partialCert *hotstuffpb.PartialCert, depth int) string { - if partialCert == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "hotstuffpb.PartialCert{\n"+ - "%s\tSig: %v\n"+ - "%s\tHash: %v\n"+ - "%s}", - tabs, QuorumSignatureToString(partialCert.Sig, depth+1), - tabs, partialCert.Hash, - tabs) -} - -func QuorumSignatureToString(quorumSignature *hotstuffpb.QuorumSignature, depth int) string { - if quorumSignature == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - sigString := "" - if quorumSignature.Sig == nil { - sigString = "nil" - } else { - switch quorumSignature.Sig.(type) { - case *hotstuffpb.QuorumSignature_ECDSASigs: - sigString = QuorumSignature_ECDSASigsToString(quorumSignature.Sig.(*hotstuffpb.QuorumSignature_ECDSASigs), depth+1) - case *hotstuffpb.QuorumSignature_BLS12Sig: - sigString = QuorumSignature_BLS12SigToString(quorumSignature.Sig.(*hotstuffpb.QuorumSignature_BLS12Sig), depth+1) - } - } - - return fmt.Sprintf( - "&hotstuffpb.QuorumSignature{\n"+ - "%s\tSig: %s\n"+ - "%s}", - tabs, sigString, - tabs) -} - -func QuorumSignature_ECDSASigsToString(ECDSASigs *hotstuffpb.QuorumSignature_ECDSASigs, depth int) string { - if ECDSASigs == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "&hotstuffpb.QuorumSignature_ECDSASigs{\n"+ - "%s\tECDSASigs: %s\n"+ - "%s},", - tabs, ECDSAMultiSignatureToString(ECDSASigs.ECDSASigs, depth+1), - tabs) -} - -func ECDSAMultiSignatureToString(ECDSASigs *hotstuffpb.ECDSAMultiSignature, depth int) string { - if ECDSASigs == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - sigsString := "[]*ECDSASignature{\n" - - for _, sig := range ECDSASigs.Sigs { - sigsString += fmt.Sprintf("%s\t\t%s\n", tabs, ECDSASigToString(sig, depth+2)) - } - - sigsString += tabs + "\t}" - - return fmt.Sprintf( - "&hotstuffpb.ECDSAMultiSignature{\n"+ - "%s\tECDSASigs: %s\n"+ - "%s}", - tabs, sigsString, - tabs) -} - -func ECDSASigToString(sig *hotstuffpb.ECDSASignature, depth int) string { - if sig == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "&hotstuffpb.ECDSASignature{\n"+ - "%s\tSigner: %v\n"+ - "%s\tR: %v\n"+ - "%s\tS: %v\n"+ - "%s}", - tabs, sig.Signer, - tabs, sig.R, - tabs, sig.S, - tabs) -} - -func QuorumSignature_BLS12SigToString(BLS12Sig *hotstuffpb.QuorumSignature_BLS12Sig, depth int) string { - if BLS12Sig == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "&hotstuffpb.QuorumSignature_BLS12Sig{\n"+ - "%s\tBLS12Sig: %s\n"+ - "%s},", - tabs, BLS12AggregateSignatureToString(BLS12Sig.BLS12Sig, depth+1), - tabs) -} - -func BLS12AggregateSignatureToString(BLS12Sig *hotstuffpb.BLS12AggregateSignature, depth int) string { - if BLS12Sig == nil { - return "nil" - } - - tabs := depthToTabs(depth) - - return fmt.Sprintf( - "&hotstuffpb.BLS12AggregateSignature{\n"+ - "%s\tSig: %v\n"+ - "%s\tParticipants: %v\n"+ - "%s},", - tabs, BLS12Sig.Sig, - tabs, BLS12Sig.Participants, - tabs) -} diff --git a/fuzz/types.go b/fuzz/types.go index d04bc57fa..25d3f8eb2 100644 --- a/fuzz/types.go +++ b/fuzz/types.go @@ -1,63 +1,56 @@ package fuzz import ( + "fmt" + "github.com/relab/hotstuff" "github.com/relab/hotstuff/internal/proto/hotstuffpb" + "google.golang.org/protobuf/proto" ) -type FuzzMsgInterface interface { - ToMsg() any - ToString(int) string - String() string -} - -type AlmostFuzzMsg interface { - Msg() FuzzMsgInterface -} - -func (proposeFuzzMsg *FuzzMsg_ProposeMsg) Msg() FuzzMsgInterface { - return proposeFuzzMsg.ProposeMsg -} - -func (timeoutFuzzMsg *FuzzMsg_TimeoutMsg) Msg() FuzzMsgInterface { - return timeoutFuzzMsg.TimeoutMsg -} - -func (newViewFuzzMsg *FuzzMsg_NewViewMsg) Msg() FuzzMsgInterface { - return newViewFuzzMsg.NewViewMsg -} - -func (voteFuzzMsg *FuzzMsg_VoteMsg) Msg() FuzzMsgInterface { - return voteFuzzMsg.VoteMsg -} - -func (fuzzMsg *FuzzMsg) Msg() FuzzMsgInterface { - return fuzzMsg.Message.(AlmostFuzzMsg).Msg() -} - -func (proposeFuzzMsg *ProposeMsg) ToMsg() any { - proposeMsg := hotstuffpb.ProposalFromProto(proposeFuzzMsg.Proposal) - proposeMsg.ID = hotstuff.ID(proposeFuzzMsg.ID) - return proposeMsg -} - -func (timeoutFuzzMsg *TimeoutMsg) ToMsg() any { - timeoutMsg := hotstuffpb.TimeoutMsgFromProto(timeoutFuzzMsg.TimeoutMsg) - timeoutMsg.ID = hotstuff.ID(timeoutFuzzMsg.ID) - return timeoutMsg -} - -func (voteFuzzMsg *VoteMsg) ToMsg() any { - voteMsg := hotstuff.VoteMsg{} - voteMsg.PartialCert = hotstuffpb.PartialCertFromProto(voteFuzzMsg.PartialCert) - voteMsg.ID = hotstuff.ID(voteFuzzMsg.ID) - voteMsg.Deferred = voteFuzzMsg.Deferred - return voteMsg -} - -func (newViewFuzzMsg *NewViewMsg) ToMsg() any { - newViewMsg := hotstuff.NewViewMsg{} - newViewMsg.SyncInfo = hotstuffpb.SyncInfoFromProto(newViewFuzzMsg.SyncInfo) - newViewMsg.ID = hotstuff.ID(newViewFuzzMsg.ID) - return newViewMsg +// extractProtoMsg extracts the proto message from the FuzzMsg's oneof field. +func extractProtoMsg(fuzzMsg *FuzzMsg) (m proto.Message) { + switch msg := fuzzMsg.Message.(type) { + case *FuzzMsg_ProposeMsg: + return msg.ProposeMsg + case *FuzzMsg_TimeoutMsg: + return msg.TimeoutMsg + case *FuzzMsg_VoteMsg: + return msg.VoteMsg + case *FuzzMsg_NewViewMsg: + return msg.NewViewMsg + default: + panic(fmt.Errorf("unknown message type: %T", msg)) + } +} + +// fuzzMsgToHotStuffMsg returns a hotstuff message extracted from a FuzzMsg encapsulating a proto message. +func fuzzMsgToHotStuffMsg(fuzzMsg *FuzzMsg) any { + switch msg := fuzzMsg.Message.(type) { + case *FuzzMsg_ProposeMsg: + proposeMsg := hotstuffpb.ProposalFromProto(msg.ProposeMsg.Proposal) + proposeMsg.ID = hotstuff.ID(msg.ProposeMsg.ID) + return proposeMsg + + case *FuzzMsg_TimeoutMsg: + timeoutMsg := hotstuffpb.TimeoutMsgFromProto(msg.TimeoutMsg.TimeoutMsg) + timeoutMsg.ID = hotstuff.ID(msg.TimeoutMsg.ID) + return timeoutMsg + + case *FuzzMsg_VoteMsg: + voteMsg := hotstuff.VoteMsg{} + voteMsg.PartialCert = hotstuffpb.PartialCertFromProto(msg.VoteMsg.PartialCert) + voteMsg.ID = hotstuff.ID(msg.VoteMsg.ID) + voteMsg.Deferred = msg.VoteMsg.Deferred + return voteMsg + + case *FuzzMsg_NewViewMsg: + newViewMsg := hotstuff.NewViewMsg{} + newViewMsg.SyncInfo = hotstuffpb.SyncInfoFromProto(msg.NewViewMsg.SyncInfo) + newViewMsg.ID = hotstuff.ID(msg.NewViewMsg.ID) + return newViewMsg + + default: + panic(fmt.Errorf("unknown message type: %T", msg)) + } } diff --git a/go.mod b/go.mod index 363e34f25..c2abcad5e 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/google/gofuzz v1.1.0 github.com/kilic/bls12-381 v0.1.1-0.20210208205449-6045b0235e36 github.com/mattn/go-isatty v0.0.14 + github.com/meling/proto2 v0.0.0-20230629054207-f46066bf57ea github.com/mitchellh/go-homedir v1.1.0 github.com/mroth/weightedrand v0.4.1 github.com/relab/gorums v0.7.1-0.20220307181651-94a8af8e467c @@ -22,7 +23,7 @@ require ( gonum.org/v1/plot v0.11.0 google.golang.org/genproto v0.0.0-20220525015930-6ca3db687a9d google.golang.org/grpc v1.47.0 - google.golang.org/protobuf v1.28.0 + google.golang.org/protobuf v1.31.0 ) require ( @@ -43,7 +44,7 @@ require ( github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/gonuts/binary v0.2.0 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/pprof v0.0.0-20220412212628-83db2b799d1f // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect diff --git a/go.sum b/go.sum index 4bd2dcd1f..44606e063 100644 --- a/go.sum +++ b/go.sum @@ -395,8 +395,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -510,6 +510,8 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/meling/proto2 v0.0.0-20230629054207-f46066bf57ea h1:3zx2Jl9M4IoFVxfBmH7I8hyWZY5NboQdG00n1NJRDqs= +github.com/meling/proto2 v0.0.0-20230629054207-f46066bf57ea/go.mod h1:pvWGo3hsDXoNnRuPLpXVOZj4ABerp8yPbFG2EyDeKno= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -1190,8 +1192,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 700ddc868eddec5eb5da00856101dbbd273e4ef9 Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Thu, 29 Jun 2023 15:53:02 -0700 Subject: [PATCH 13/25] Unexported a bunch of methods and types to avoid lint warnings Also, there is no need to export these types at this point, since they are not being used from outside the fuzz package. --- fuzz/doc.go | 2 ++ fuzz/error.go | 88 +++++++++++++++++++++++----------------------- fuzz/extra_test.go | 8 ++--- fuzz/fuzz_test.go | 54 ++++++++++++++-------------- fuzz/serialize.go | 3 +- 5 files changed, 78 insertions(+), 77 deletions(-) create mode 100644 fuzz/doc.go diff --git a/fuzz/doc.go b/fuzz/doc.go new file mode 100644 index 000000000..19a7eae5e --- /dev/null +++ b/fuzz/doc.go @@ -0,0 +1,2 @@ +// Package fuzz contains fuzzing functions for the fuzzing protobuf/hotstuff messages. +package fuzz diff --git a/fuzz/error.go b/fuzz/error.go index 76ec1175a..1f9965f06 100644 --- a/fuzz/error.go +++ b/fuzz/error.go @@ -11,15 +11,15 @@ import ( "github.com/meling/proto2" ) -type TypeCount map[reflect.Type]int +type typeCount map[reflect.Type]int -func (typeCount TypeCount) Add(typ reflect.Type) { - typeCount[typ]++ +func (tc typeCount) Add(typ reflect.Type) { + tc[typ]++ } -func (typeCount TypeCount) String(typeTotalCount TypeCount) string { +func (tc typeCount) String(typeTotalCount typeCount) string { keys := make([]reflect.Type, 0) - for key := range typeCount { + for key := range tc { keys = append(keys, key) } @@ -27,43 +27,43 @@ func (typeCount TypeCount) String(typeTotalCount TypeCount) string { str := "" for _, key := range keys { - str += key.String() + ": " + strconv.Itoa(typeCount[key]) + " / " + strconv.Itoa(typeTotalCount[key]) + "\n" + str += key.String() + ": " + strconv.Itoa(tc[key]) + " / " + strconv.Itoa(typeTotalCount[key]) + "\n" } return str } -type PanicInfo struct { +type panicInfo struct { Err any StackTrace string FuzzMsg string FuzzMsgB64 string Seed *int64 LineNum int - TypeCount TypeCount + TypeCount typeCount } -type ErrorInfo struct { +type errorInfo struct { messageFile string currentFuzzMsg *FuzzMsg currentFuzzMsgB64 string currentFuzzMsgSeed *int64 errorCount int - panics map[string]PanicInfo + panics map[string]panicInfo totalScenarios int failedScenarios int totalMessages int failedMessages int - TypePanicCount TypeCount - TypeTotalCount TypeCount + TypePanicCount typeCount + TypeTotalCount typeCount } -func (errorInfo *ErrorInfo) Init() { - errorInfo.panics = make(map[string]PanicInfo) - errorInfo.TypeTotalCount = make(TypeCount) - errorInfo.TypePanicCount = make(TypeCount) +func (ei *errorInfo) init() { + ei.panics = make(map[string]panicInfo) + ei.TypeTotalCount = make(typeCount) + ei.TypePanicCount = make(typeCount) } -func (errorInfo *ErrorInfo) OutputInfo(t *testing.T) { +func (ei *errorInfo) outputInfo(t *testing.T) { b64s := "" seeds := "" @@ -74,7 +74,7 @@ func (errorInfo *ErrorInfo) OutputInfo(t *testing.T) { fmt.Println("ERROR INFO") keys := make([]string, 0) - for key := range errorInfo.panics { + for key := range ei.panics { keys = append(keys, key) } @@ -82,7 +82,7 @@ func (errorInfo *ErrorInfo) OutputInfo(t *testing.T) { sort.Strings(keys) for i, key := range keys { - panicInfo := errorInfo.panics[key] + panicInfo := ei.panics[key] b64s += panicInfo.FuzzMsgB64 + "\n" if panicInfo.Seed != nil { @@ -95,7 +95,7 @@ func (errorInfo *ErrorInfo) OutputInfo(t *testing.T) { fmt.Println(key) fmt.Println() fmt.Println("crash amounts grouped by type:") - fmt.Println(panicInfo.TypeCount.String(errorInfo.TypeTotalCount)) + fmt.Println(panicInfo.TypeCount.String(ei.TypeTotalCount)) fmt.Println("- STACK TRACE BEGIN") fmt.Print(panicInfo.StackTrace) fmt.Println("- STACK TRACE END") @@ -118,66 +118,66 @@ func (errorInfo *ErrorInfo) OutputInfo(t *testing.T) { fmt.Println() fmt.Println("SUMMARY") - fmt.Printf("unique errors found: %d\n", len(errorInfo.panics)) - fmt.Printf("%d runs were errors\n", errorInfo.errorCount) - fmt.Printf("%d of %d scenarios failed\n", errorInfo.failedScenarios, errorInfo.totalScenarios) - fmt.Printf("%d of %d messages failed\n", errorInfo.failedMessages, errorInfo.totalMessages) + fmt.Printf("unique errors found: %d\n", len(ei.panics)) + fmt.Printf("%d runs were errors\n", ei.errorCount) + fmt.Printf("%d of %d scenarios failed\n", ei.failedScenarios, ei.totalScenarios) + fmt.Printf("%d of %d messages failed\n", ei.failedMessages, ei.totalMessages) fmt.Println() fmt.Println("crash amounts grouped by type:") - fmt.Println(errorInfo.TypePanicCount.String(errorInfo.TypeTotalCount)) + fmt.Println(ei.TypePanicCount.String(ei.TypeTotalCount)) } -func (errorInfo *ErrorInfo) AddTotal(fuzzMessage *FuzzMsg, seed *int64) { - errorInfo.totalMessages++ - errorInfo.currentFuzzMsg = fuzzMessage - errorInfo.currentFuzzMsgSeed = seed - protoMsg := extractProtoMsg(errorInfo.currentFuzzMsg) +func (ei *errorInfo) addTotal(fuzzMessage *FuzzMsg, seed *int64) { + ei.totalMessages++ + ei.currentFuzzMsg = fuzzMessage + ei.currentFuzzMsgSeed = seed + protoMsg := extractProtoMsg(ei.currentFuzzMsg) typ := reflect.TypeOf(protoMsg) - errorInfo.TypeTotalCount.Add(typ) + ei.TypeTotalCount.Add(typ) } -func (errorInfo *ErrorInfo) AddPanic(fullStack string, err2 any, info string) { - simpleStack := SimplifyStack(fullStack) +func (ei *errorInfo) addPanic(fullStack string, err2 any, info string) { + simpleStack := simplifyStack(fullStack) identifier := "error location:\t" + simpleStack + "\nerror info:\t" + fmt.Sprint(err2) + "\nrecovered from:\t" + info - errorInfo.errorCount++ + ei.errorCount++ - oldPanic, okPanic := errorInfo.panics[identifier] + oldPanic, okPanic := ei.panics[identifier] - b64, err := fuzzMsgToB64(errorInfo.currentFuzzMsg) + b64, err := fuzzMsgToB64(ei.currentFuzzMsg) if err != nil { panic(err) } - protoMsg := extractProtoMsg(errorInfo.currentFuzzMsg) + protoMsg := extractProtoMsg(ei.currentFuzzMsg) fuzzMsgString := proto2.GoString(protoMsg) newLines := strings.Count(fuzzMsgString, "\n") - newPanic := PanicInfo{ + newPanic := panicInfo{ Err: err2, StackTrace: fullStack, FuzzMsg: fuzzMsgString, FuzzMsgB64: b64, - Seed: errorInfo.currentFuzzMsgSeed, + Seed: ei.currentFuzzMsgSeed, LineNum: newLines, } if okPanic { newPanic.TypeCount = oldPanic.TypeCount } else { - newPanic.TypeCount = make(TypeCount) + newPanic.TypeCount = make(typeCount) } oldLines := oldPanic.LineNum if !okPanic || newLines < oldLines { - errorInfo.panics[identifier] = newPanic + ei.panics[identifier] = newPanic } typ := reflect.TypeOf(protoMsg) - errorInfo.panics[identifier].TypeCount.Add(typ) - errorInfo.TypePanicCount.Add(typ) + ei.panics[identifier].TypeCount.Add(typ) + ei.TypePanicCount.Add(typ) } -func SimplifyStack(stack string) string { +func simplifyStack(stack string) string { stackLines := strings.Split(strings.ReplaceAll(stack, "\r\n", "\n"), "\n") // line 9 tells us where the panic happened, found through testing return stackLines[8][1:] diff --git a/fuzz/extra_test.go b/fuzz/extra_test.go index aeeca4267..49c26a55e 100644 --- a/fuzz/extra_test.go +++ b/fuzz/extra_test.go @@ -16,8 +16,8 @@ func TestFrequencyErrorFuzz(t *testing.T) { f := initFuzz() for j := 0; j < 1000; j++ { - errorInfo := new(ErrorInfo) - errorInfo.Init() + errorInfo := new(errorInfo) + errorInfo.init() iterations := 1 @@ -44,8 +44,8 @@ func TestFrequencyErrorFuzz(t *testing.T) { } func BenchmarkFuzz(b *testing.B) { - errorInfo := new(ErrorInfo) - errorInfo.Init() + errorInfo := new(errorInfo) + errorInfo.init() f := initFuzz() diff --git a/fuzz/fuzz_test.go b/fuzz/fuzz_test.go index 1adb9bedc..93186ee15 100644 --- a/fuzz/fuzz_test.go +++ b/fuzz/fuzz_test.go @@ -9,14 +9,14 @@ import ( _ "github.com/relab/hotstuff/consensus/chainedhotstuff" ) -func TryExecuteScenario(errorInfo *ErrorInfo, oldMessage any, newMessage any) { - errorInfo.totalScenarios++ +func TryExecuteScenario(errInfo *errorInfo, oldMessage any, newMessage any) { + errInfo.totalScenarios++ defer func() { if err := recover(); err != nil { stack := string(debug.Stack()) - errorInfo.AddPanic(stack, err, "TryExecuteScenario") - errorInfo.failedScenarios++ + errInfo.addPanic(stack, err, "TryExecuteScenario") + errInfo.failedScenarios++ } }() @@ -68,35 +68,35 @@ func getMessagesBasicScenario() int { return messageCount } -func fuzzScenario(errorInfo *ErrorInfo, newMessage any) { - TryExecuteScenario(errorInfo, 1, newMessage) +func fuzzScenario(errInfo *errorInfo, newMessage any) { + TryExecuteScenario(errInfo, 1, newMessage) } -func fuzzMsgToMsg(errorInfo *ErrorInfo, fuzzMsg *FuzzMsg) any { - errorInfo.totalMessages++ +func fuzzMsgToMsg(errInfo *errorInfo, fuzzMsg *FuzzMsg) any { + errInfo.totalMessages++ defer func() { if err := recover(); err != nil { stack := string(debug.Stack()) - errorInfo.AddPanic(stack, err, "fuzzMsgToMsg") - errorInfo.failedMessages++ + errInfo.addPanic(stack, err, "fuzzMsgToMsg") + errInfo.failedMessages++ } }() return fuzzMsgToHotStuffMsg(fuzzMsg) } -func useFuzzMessage(errorInfo *ErrorInfo, fuzzMessage *FuzzMsg, seed *int64) { - errorInfo.AddTotal(fuzzMessage, seed) - newMessage := fuzzMsgToMsg(errorInfo, fuzzMessage) +func useFuzzMessage(errInfo *errorInfo, fuzzMessage *FuzzMsg, seed *int64) { + errInfo.addTotal(fuzzMessage, seed) + newMessage := fuzzMsgToMsg(errInfo, fuzzMessage) if newMessage != nil { - fuzzScenario(errorInfo, newMessage) + fuzzScenario(errInfo, newMessage) } } // the main test func TestFuzz(t *testing.T) { - errorInfo := new(ErrorInfo) - errorInfo.Init() + errInfo := new(errorInfo) + errInfo.init() f := initFuzz() @@ -105,21 +105,21 @@ func TestFuzz(t *testing.T) { for i := 0; i < iterations; i++ { // \r is carriage return, writing the // next line will overwrite the previous ;) - fmt.Printf("running test %4d/%4d %4d errors \r", i+1, iterations, errorInfo.errorCount) + fmt.Printf("running test %4d/%4d %4d errors \r", i+1, iterations, errInfo.errorCount) seed := rand.Int63() fuzzMessage := createFuzzMessage(f, &seed) - useFuzzMessage(errorInfo, fuzzMessage, &seed) + useFuzzMessage(errInfo, fuzzMessage, &seed) } - errorInfo.OutputInfo(t) + errInfo.outputInfo(t) } // load previously created fuzz messages from a file // it doesn't work quite right, i blame proto.Marshal() func TestPreviousFuzz(t *testing.T) { - errorInfo := new(ErrorInfo) - errorInfo.Init() + errInfo := new(errorInfo) + errInfo.init() fuzzMsgs, err := loadFuzzMessagesFromFile("previous_messages.b64") if err != nil { @@ -127,17 +127,17 @@ func TestPreviousFuzz(t *testing.T) { } for _, fuzzMessage := range fuzzMsgs { - useFuzzMessage(errorInfo, fuzzMessage, nil) + useFuzzMessage(errInfo, fuzzMessage, nil) } - errorInfo.OutputInfo(t) + errInfo.outputInfo(t) } // load previously created fuzz messages from a file // it recreates the fuzz messages from a 64-bit source func TestSeedPreviousFuzz(t *testing.T) { - errorInfo := new(ErrorInfo) - errorInfo.Init() + errInfo := new(errorInfo) + errInfo.init() seeds, err := loadSeedsFromFile("previous_messages.seed") if err != nil { @@ -148,8 +148,8 @@ func TestSeedPreviousFuzz(t *testing.T) { for _, seed := range seeds { fuzzMessage := createFuzzMessage(f, &seed) - useFuzzMessage(errorInfo, fuzzMessage, nil) + useFuzzMessage(errInfo, fuzzMessage, nil) } - errorInfo.OutputInfo(t) + errInfo.outputInfo(t) } diff --git a/fuzz/serialize.go b/fuzz/serialize.go index 0a24d8fe1..bd93bf71d 100644 --- a/fuzz/serialize.go +++ b/fuzz/serialize.go @@ -48,8 +48,7 @@ func saveStringToFile(filename string, str string) error { w := bufio.NewWriter(f) _, err = w.WriteString(str) w.Flush() - - return nil + return err } func loadStringFromFile(filename string) (string, error) { From c8dc0ac2828a1b4b76b51d0f214234c7818293df Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Thu, 29 Jun 2023 16:15:30 -0700 Subject: [PATCH 14/25] Cleaned up the serialize.go code --- fuzz/serialize.go | 77 +++++++++++------------------------------------ 1 file changed, 18 insertions(+), 59 deletions(-) diff --git a/fuzz/serialize.go b/fuzz/serialize.go index bd93bf71d..4468c12f3 100644 --- a/fuzz/serialize.go +++ b/fuzz/serialize.go @@ -1,7 +1,6 @@ package fuzz import ( - "bufio" "encoding/base64" "os" "strconv" @@ -12,63 +11,35 @@ import ( func fuzzMsgToB64(msg *FuzzMsg) (string, error) { bytes, err := proto.Marshal(msg) - str := base64.StdEncoding.EncodeToString(bytes) - - return str, err + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(bytes), nil } func b64ToFuzzMsg(str string) (*FuzzMsg, error) { bytes, err := base64.StdEncoding.DecodeString(str) - + if err != nil { + return nil, err + } msg := new(FuzzMsg) - proto.Unmarshal(bytes, msg) - + err = proto.Unmarshal(bytes, msg) + if err != nil { + return nil, err + } return msg, err } -/*func b64TofuzzMessages(str string) ([]FuzzMsg, error) { - - strLines := strings.Split(strings.ReplaceAll(str, "\r\n", "\n"), "\n") - for _, msg := range messages { - str, err := fuzzMsgToB64(msg) - if err != nil { - return nil, err - } - } - - return full_str, nil -}*/ - func saveStringToFile(filename string, str string) error { - f, err := os.Create(filename) - if err != nil { - return err - } - defer f.Close() - w := bufio.NewWriter(f) - _, err = w.WriteString(str) - w.Flush() - return err + return os.WriteFile(filename, []byte(str), 0o600) } func loadStringFromFile(filename string) (string, error) { - f, err := os.Open(filename) - if err != nil { - return "", err - } - - fi, err := f.Stat() + b, err := os.ReadFile(filename) if err != nil { return "", err } - - bytes := make([]byte, fi.Size()) - - f.Read(bytes) - - b64string := string(bytes) - - return b64string, nil + return string(b), nil } func loadFuzzMessagesFromFile(filename string) ([]*FuzzMsg, error) { @@ -76,24 +47,17 @@ func loadFuzzMessagesFromFile(filename string) ([]*FuzzMsg, error) { if err != nil { return nil, err } - - b64s := strings.Split(str, "\n") - fuzzMsgs := make([]*FuzzMsg, 0) - - for _, b64 := range b64s { - + for _, b64 := range strings.Split(str, "\n") { if b64 == "" { continue } - fuzzMsg, err := b64ToFuzzMsg(b64) if err != nil { return nil, err } fuzzMsgs = append(fuzzMsgs, fuzzMsg) } - return fuzzMsgs, nil } @@ -102,21 +66,16 @@ func loadSeedsFromFile(filename string) ([]int64, error) { if err != nil { return nil, err } - - seedstrs := strings.Split(str, "\n") seeds := make([]int64, 0) - - for _, seedstr := range seedstrs { - if seedstr == "" { + for _, seedString := range strings.Split(str, "\n") { + if seedString == "" { continue } - - seed, err := strconv.ParseInt(seedstr, 10, 64) + seed, err := strconv.ParseInt(seedString, 10, 64) if err != nil { return nil, err } seeds = append(seeds, seed) } - return seeds, nil } From 16d50a1f10105e3e73bde5ab70460a33115ee614 Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Thu, 29 Jun 2023 16:23:54 -0700 Subject: [PATCH 15/25] Removed experimental_string.go; superceded by proto2.GoString() --- fuzz/experimental_string.go | 141 ------------------------------------ fuzz/extra_test.go | 35 ++------- fuzz/fuzz_test.go | 8 +- 3 files changed, 10 insertions(+), 174 deletions(-) delete mode 100644 fuzz/experimental_string.go diff --git a/fuzz/experimental_string.go b/fuzz/experimental_string.go deleted file mode 100644 index e4aa20fde..000000000 --- a/fuzz/experimental_string.go +++ /dev/null @@ -1,141 +0,0 @@ -package fuzz - -import ( - "fmt" - - "google.golang.org/protobuf/reflect/protoreflect" -) - -func depthToTabs(depth int) (tabs string) { - for i := 0; i < depth; i++ { - tabs += "\t" - } - return -} - -// work in progress -func interfaceToString(i interface{}) string { - return fmt.Sprintf("%v", i) -} - -func valueToString(v protoreflect.Value, depth int) string { - _, ok := v.Interface().(protoreflect.Message) - if ok { - return protoToString(v.Message(), depth+1) - } - return interfaceToString(v.Interface()) -} - -func listToString(list protoreflect.List, depth int) string { - if list.Len() == 0 { - return "[]" - } - - tabs := depthToTabs(depth) - - str := "[\n" - - for i := 0; i < list.Len(); i++ { - item := list.Get(i) - str += tabs + "\t" + valueToString(item, depth) + ",\n" - } - - str += tabs + "]" - - return str -} - -func mapToString(mp protoreflect.Map, depth int) string { - if mp.Len() == 0 { - return "[]" - } - - tabs := depthToTabs(depth) - str := "[\n" - - mp.Range(func(mk protoreflect.MapKey, v protoreflect.Value) bool { - str += tabs + "\t" + valueToString(mk.Value(), depth) + ": " + valueToString(v, depth) + ",\n" - return true - }) - - str += tabs + "]" - return str -} - -func fieldToString(field protoreflect.FieldDescriptor, prm protoreflect.Message, depth int) string { - msg := field.Message() - - str := "" - - if msg == nil { - return str + prm.Get(field).String() + "\n" - } - - if field.IsList() { - list := prm.Get(field).List() - return str + listToString(list, depth) + "\n" - } - - if field.IsExtension() { - panic("extension to string not implemented") - } - - if field.IsMap() { - map2 := prm.Get(field).Map() - return str + mapToString(map2, depth) + "\n" - } - - if field.IsPlaceholder() { - panic("placeholder to string not implemented") - } - - refl := prm.Get(field).Message() - if refl == nil { - panic("message reflection is nil") - } - return str + protoToString(refl, depth) + "\n" -} - -func protoToString(prm protoreflect.Message, depth int) string { - if !prm.IsValid() { - return "nil\n" - } - - desc := prm.Descriptor() - tabs := depthToTabs(depth) - str := string(desc.FullName()) + "{\n" - fields := desc.Fields() - - for i := 0; i < fields.Len(); i++ { - field := fields.Get(i) - - if field.ContainingOneof() != nil { - continue - } - - str += tabs + "\t" + string(field.Name()) + ": " + fieldToString(field, prm, depth+1) - } - - oneofs := desc.Oneofs() - - for i := 0; i < oneofs.Len(); i++ { - oneof := oneofs.Get(i) - - fields := oneof.Fields() - - for j := 0; j < fields.Len(); j++ { - field := fields.Get(j) - - refl := prm.Get(field).Message() - if !refl.IsValid() { - continue - } - - str += tabs + "\t" + string(oneof.Name()) + ": " + fieldToString(field, prm, depth+1) - } - } - - str += tabs + "}" - - return str -} diff --git a/fuzz/extra_test.go b/fuzz/extra_test.go index 49c26a55e..43a34f757 100644 --- a/fuzz/extra_test.go +++ b/fuzz/extra_test.go @@ -7,8 +7,6 @@ import ( "fmt" "math/rand" "testing" - - "google.golang.org/protobuf/proto" ) func TestFrequencyErrorFuzz(t *testing.T) { @@ -16,17 +14,17 @@ func TestFrequencyErrorFuzz(t *testing.T) { f := initFuzz() for j := 0; j < 1000; j++ { - errorInfo := new(errorInfo) - errorInfo.init() + errInfo := new(errorInfo) + errInfo.init() iterations := 1 for i := 0; i < iterations; i++ { fuzzMessage := createFuzzMessage(f, nil) - useFuzzMessage(errorInfo, fuzzMessage, nil) + useFuzzMessage(errInfo, fuzzMessage, nil) } - for key := range errorInfo.panics { + for key := range errInfo.panics { frequency[key]++ } } @@ -44,36 +42,17 @@ func TestFrequencyErrorFuzz(t *testing.T) { } func BenchmarkFuzz(b *testing.B) { - errorInfo := new(errorInfo) - errorInfo.init() + errInfo := new(errorInfo) + errInfo.init() f := initFuzz() for i := 0; i < b.N; i++ { seed := rand.Int63() fuzzMessage := createFuzzMessage(f, &seed) - useFuzzMessage(errorInfo, fuzzMessage, &seed) + useFuzzMessage(errInfo, fuzzMessage, &seed) } fmt.Println(b.Elapsed()) - - // errorInfo.OutputInfo(nil) - fmt.Println() } - -func TestExperimentalString(t *testing.T) { - f := initFuzz() - fuzzMessage := createFuzzMessage(f, nil) - - /*fuzzMessage := hotstuffpb.ECDSASignature{ - Signer: uint32(10), - R: []byte{3, 1, 2}, - S: []byte{5, 4, 3}, - }*/ - - var msg proto.Message - msg = fuzzMessage - - fmt.Println(protoToString(msg.ProtoReflect(), 0)) -} diff --git a/fuzz/fuzz_test.go b/fuzz/fuzz_test.go index 93186ee15..b65e29422 100644 --- a/fuzz/fuzz_test.go +++ b/fuzz/fuzz_test.go @@ -9,7 +9,7 @@ import ( _ "github.com/relab/hotstuff/consensus/chainedhotstuff" ) -func TryExecuteScenario(errInfo *errorInfo, oldMessage any, newMessage any) { +func tryExecuteScenario(errInfo *errorInfo, oldMessage any, newMessage any) { errInfo.totalScenarios++ defer func() { if err := recover(); err != nil { @@ -63,13 +63,11 @@ func getMessagesBasicScenario() int { result, _ := ExecuteScenario(s, numNodes, 0, 100, "chainedhotstuff") - messageCount := result.MessageCount - - return messageCount + return result.MessageCount } func fuzzScenario(errInfo *errorInfo, newMessage any) { - TryExecuteScenario(errInfo, 1, newMessage) + tryExecuteScenario(errInfo, 1, newMessage) } func fuzzMsgToMsg(errInfo *errorInfo, fuzzMsg *FuzzMsg) any { From 0906eae16b9674a6de07d903ef918b4e4c5d0bcd Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Mon, 3 Jul 2023 12:06:56 -0700 Subject: [PATCH 16/25] Updated API to proto2.GoStruct() --- fuzz/error.go | 2 +- go.mod | 2 +- go.sum | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/fuzz/error.go b/fuzz/error.go index 1f9965f06..c0fcfa8af 100644 --- a/fuzz/error.go +++ b/fuzz/error.go @@ -150,7 +150,7 @@ func (ei *errorInfo) addPanic(fullStack string, err2 any, info string) { } protoMsg := extractProtoMsg(ei.currentFuzzMsg) - fuzzMsgString := proto2.GoString(protoMsg) + fuzzMsgString := proto2.GoStruct(protoMsg) newLines := strings.Count(fuzzMsgString, "\n") newPanic := panicInfo{ diff --git a/go.mod b/go.mod index c2abcad5e..cb10115b3 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/google/gofuzz v1.1.0 github.com/kilic/bls12-381 v0.1.1-0.20210208205449-6045b0235e36 github.com/mattn/go-isatty v0.0.14 - github.com/meling/proto2 v0.0.0-20230629054207-f46066bf57ea + github.com/meling/proto2 v0.0.0-20230703190343-27bbb7955179 github.com/mitchellh/go-homedir v1.1.0 github.com/mroth/weightedrand v0.4.1 github.com/relab/gorums v0.7.1-0.20220307181651-94a8af8e467c diff --git a/go.sum b/go.sum index 44606e063..f6dfbebf8 100644 --- a/go.sum +++ b/go.sum @@ -512,6 +512,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/meling/proto2 v0.0.0-20230629054207-f46066bf57ea h1:3zx2Jl9M4IoFVxfBmH7I8hyWZY5NboQdG00n1NJRDqs= github.com/meling/proto2 v0.0.0-20230629054207-f46066bf57ea/go.mod h1:pvWGo3hsDXoNnRuPLpXVOZj4ABerp8yPbFG2EyDeKno= +github.com/meling/proto2 v0.0.0-20230703190343-27bbb7955179 h1:ywfmQjNyaL0LrfjkN0rir8LvRvLThhupx/kXDbpHb7c= +github.com/meling/proto2 v0.0.0-20230703190343-27bbb7955179/go.mod h1:pvWGo3hsDXoNnRuPLpXVOZj4ABerp8yPbFG2EyDeKno= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -1270,6 +1272,7 @@ k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= From ca89744c8af5dce42aafdfa08f17026f4ed97652 Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Mon, 3 Jul 2023 12:31:48 -0700 Subject: [PATCH 17/25] Fixed compile error due to merge master --- fuzz/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/network.go b/fuzz/network.go index 27f08d84a..5b3400a3e 100644 --- a/fuzz/network.go +++ b/fuzz/network.go @@ -187,7 +187,7 @@ func (n *Network) tick() { for _, node := range n.nodes { node.eventLoop.AddEvent(tick{}) // run each event loop as long as it has events - for node.eventLoop.Tick() { + for node.eventLoop.Tick(context.Background()) { } } } From 2662b710066fb203af72d6397ad6c9ec09b2f263 Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Mon, 3 Jul 2023 16:09:03 -0700 Subject: [PATCH 18/25] Removed various fields and arguments to avoid lint issues --- .vscode/dict.txt | 2 ++ fuzz/error.go | 17 ++++++++++------- fuzz/extra_test.go | 7 ++++--- fuzz/fuzz_test.go | 19 ------------------- fuzz/network.go | 4 ++-- fuzz/scenario.go | 2 +- 6 files changed, 19 insertions(+), 32 deletions(-) diff --git a/.vscode/dict.txt b/.vscode/dict.txt index 35b541d42..f39df6f55 100644 --- a/.vscode/dict.txt +++ b/.vscode/dict.txt @@ -24,7 +24,9 @@ Feng ferr fgprof fgprofprofile +Fuzzer gocyclo +gofuzz golangci golint gomock diff --git a/fuzz/error.go b/fuzz/error.go index c0fcfa8af..bdbb78018 100644 --- a/fuzz/error.go +++ b/fuzz/error.go @@ -43,9 +43,7 @@ type panicInfo struct { } type errorInfo struct { - messageFile string currentFuzzMsg *FuzzMsg - currentFuzzMsgB64 string currentFuzzMsgSeed *int64 errorCount int panics map[string]panicInfo @@ -64,6 +62,7 @@ func (ei *errorInfo) init() { } func (ei *errorInfo) outputInfo(t *testing.T) { + t.Helper() b64s := "" seeds := "" @@ -105,15 +104,19 @@ func (ei *errorInfo) outputInfo(t *testing.T) { fmt.Println("- FUZZ MESSAGE END") fmt.Println() - if t != nil { - t.Error(panicInfo.Err) - } + t.Error(panicInfo.Err) } - saveStringToFile("previous_messages.b64", b64s) + err := saveStringToFile("previous_messages.b64", b64s) + if err != nil { + t.Error(err) + } if seeds != "" { - saveStringToFile("previous_messages.seed", seeds) + err := saveStringToFile("previous_messages.seed", seeds) + if err != nil { + t.Error(err) + } } fmt.Println() diff --git a/fuzz/extra_test.go b/fuzz/extra_test.go index 43a34f757..760af2777 100644 --- a/fuzz/extra_test.go +++ b/fuzz/extra_test.go @@ -6,10 +6,14 @@ package fuzz import ( "fmt" "math/rand" + "os" "testing" ) func TestFrequencyErrorFuzz(t *testing.T) { + if os.Getenv("FUZZ") == "" { + t.Skip("Skipping slow test; run with FUZZ=1 to enable") + } frequency := make(map[string]int, 0) f := initFuzz() @@ -18,7 +22,6 @@ func TestFrequencyErrorFuzz(t *testing.T) { errInfo.init() iterations := 1 - for i := 0; i < iterations; i++ { fuzzMessage := createFuzzMessage(f, nil) useFuzzMessage(errInfo, fuzzMessage, nil) @@ -30,14 +33,12 @@ func TestFrequencyErrorFuzz(t *testing.T) { } sum := 0 - for key, val := range frequency { sum += val fmt.Println(key) fmt.Println(val) fmt.Println() } - fmt.Println(sum) } diff --git a/fuzz/fuzz_test.go b/fuzz/fuzz_test.go index b65e29422..9c7bc92ed 100644 --- a/fuzz/fuzz_test.go +++ b/fuzz/fuzz_test.go @@ -47,25 +47,6 @@ func tryExecuteScenario(errInfo *errorInfo, oldMessage any, newMessage any) { } } -func getMessagesBasicScenario() int { - var numNodes uint8 = 4 - - allNodesSet := make(NodeSet) - for i := 1; i <= int(numNodes); i++ { - allNodesSet.Add(uint32(i)) - } - - s := Scenario{} - s = append(s, View{Leader: 1, Partitions: []NodeSet{allNodesSet}}) - s = append(s, View{Leader: 1, Partitions: []NodeSet{allNodesSet}}) - s = append(s, View{Leader: 1, Partitions: []NodeSet{allNodesSet}}) - s = append(s, View{Leader: 1, Partitions: []NodeSet{allNodesSet}}) - - result, _ := ExecuteScenario(s, numNodes, 0, 100, "chainedhotstuff") - - return result.MessageCount -} - func fuzzScenario(errInfo *errorInfo, newMessage any) { tryExecuteScenario(errInfo, 1, newMessage) } diff --git a/fuzz/network.go b/fuzz/network.go index 5b3400a3e..740f11720 100644 --- a/fuzz/network.go +++ b/fuzz/network.go @@ -127,7 +127,7 @@ func (n *Network) GetNodeBuilder(id NodeID, pk hotstuff.PrivateKey) modules.Buil return builder } -func (n *Network) createTwinsNodes(nodes []NodeID, scenario Scenario, consensusName string) error { +func (n *Network) createTwinsNodes(nodes []NodeID, _ Scenario, consensusName string) error { cg := &commandGenerator{} for _, nodeID := range nodes { @@ -229,7 +229,7 @@ func (n *Network) shouldDrop(sender, receiver uint32, message any) bool { return ok } -func (n *Network) shouldSwap(message any) bool { +func (n *Network) shouldSwap(_ any) bool { n.logger.Infof("is %d equal to %d?", n.OldMessage, n.MessageCounter) return n.OldMessage == n.MessageCounter } diff --git a/fuzz/scenario.go b/fuzz/scenario.go index 9e6a05d51..139dc8dde 100644 --- a/fuzz/scenario.go +++ b/fuzz/scenario.go @@ -218,4 +218,4 @@ func (cm commandModule) Exec(block *hotstuff.Block) { cm.node.executedBlocks = append(cm.node.executedBlocks, block) } -func (commandModule) Fork(block *hotstuff.Block) {} +func (commandModule) Fork(_ *hotstuff.Block) {} From 725268f11c20d2c484c056835c4582c243af376a Mon Sep 17 00:00:00 2001 From: abje Date: Wed, 5 Jul 2023 12:38:34 +0200 Subject: [PATCH 19/25] show progress in a test helper function --- fuzz/fuzz_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fuzz/fuzz_test.go b/fuzz/fuzz_test.go index 9c7bc92ed..05ff0d1ef 100644 --- a/fuzz/fuzz_test.go +++ b/fuzz/fuzz_test.go @@ -72,6 +72,11 @@ func useFuzzMessage(errInfo *errorInfo, fuzzMessage *FuzzMsg, seed *int64) { } } +func ShowProgress(t *testing.T, numIteration int, iterations int, errorCount int) { + t.Helper() + fmt.Printf("running test %4d/%4d %4d errors \r", numIteration, iterations, errorCount) +} + // the main test func TestFuzz(t *testing.T) { errInfo := new(errorInfo) @@ -82,9 +87,7 @@ func TestFuzz(t *testing.T) { iterations := 1000 for i := 0; i < iterations; i++ { - // \r is carriage return, writing the - // next line will overwrite the previous ;) - fmt.Printf("running test %4d/%4d %4d errors \r", i+1, iterations, errInfo.errorCount) + ShowProgress(t, i+1, iterations, errInfo.errorCount) seed := rand.Int63() fuzzMessage := createFuzzMessage(f, &seed) From d98afd2f40da39d961dfe29f338219620ff213f0 Mon Sep 17 00:00:00 2001 From: abje Date: Wed, 5 Jul 2023 12:44:43 +0200 Subject: [PATCH 20/25] removing unneccesary argument from shouldSwap --- fuzz/network.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/fuzz/network.go b/fuzz/network.go index 740f11720..a1e5b087c 100644 --- a/fuzz/network.go +++ b/fuzz/network.go @@ -229,8 +229,7 @@ func (n *Network) shouldDrop(sender, receiver uint32, message any) bool { return ok } -func (n *Network) shouldSwap(_ any) bool { - n.logger.Infof("is %d equal to %d?", n.OldMessage, n.MessageCounter) +func (n *Network) shouldSwap() bool { return n.OldMessage == n.MessageCounter } @@ -280,11 +279,7 @@ func (c *configuration) sendMessage(id hotstuff.ID, message any) { c.network.Messages = append(c.network.Messages, message) c.network.MessageCounter++ - if c.network.shouldSwap(message) { - - /*if (c.network.NewMessage.(hotstuff.ProposeMsg).Block.Parent() == hotstuff.Hash{}) { - c.network.NewMessage.(hotstuff.ProposeMsg).Block.SetParent(message.(hotstuff.ProposeMsg).Block.Parent()) - }*/ + if c.network.shouldSwap() { c.network.logger.Infof("swapping message with fuzz message") message = c.network.NewMessage } From 31ac5c241b94b224a5161e004622248521daa41f Mon Sep 17 00:00:00 2001 From: abje Date: Wed, 5 Jul 2023 20:29:11 +0200 Subject: [PATCH 21/25] seperating additional error info from errors --- fuzz/error.go | 11 +++++++++-- fuzz/fuzz_test.go | 6 +++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/fuzz/error.go b/fuzz/error.go index bdbb78018..3aa3b3da8 100644 --- a/fuzz/error.go +++ b/fuzz/error.go @@ -61,6 +61,15 @@ func (ei *errorInfo) init() { ei.TypePanicCount = make(typeCount) } +func (ei *errorInfo) outputErrors(t *testing.T) { + + ei.outputInfo(t) + + for panicInfo := range ei.panics { + t.Error(panicInfo.Err) + } +} + func (ei *errorInfo) outputInfo(t *testing.T) { t.Helper() b64s := "" @@ -103,8 +112,6 @@ func (ei *errorInfo) outputInfo(t *testing.T) { fmt.Println(panicInfo.FuzzMsg) fmt.Println("- FUZZ MESSAGE END") fmt.Println() - - t.Error(panicInfo.Err) } err := saveStringToFile("previous_messages.b64", b64s) diff --git a/fuzz/fuzz_test.go b/fuzz/fuzz_test.go index 05ff0d1ef..17901c6ef 100644 --- a/fuzz/fuzz_test.go +++ b/fuzz/fuzz_test.go @@ -94,7 +94,7 @@ func TestFuzz(t *testing.T) { useFuzzMessage(errInfo, fuzzMessage, &seed) } - errInfo.outputInfo(t) + errInfo.outputErrors(t) } // load previously created fuzz messages from a file @@ -112,7 +112,7 @@ func TestPreviousFuzz(t *testing.T) { useFuzzMessage(errInfo, fuzzMessage, nil) } - errInfo.outputInfo(t) + errInfo.outputErrors(t) } // load previously created fuzz messages from a file @@ -133,5 +133,5 @@ func TestSeedPreviousFuzz(t *testing.T) { useFuzzMessage(errInfo, fuzzMessage, nil) } - errInfo.outputInfo(t) + errInfo.outputErrors(t) } From 91466c57fb59f54cb915d8cdafb1ae69383c11de Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Wed, 5 Jul 2023 16:30:08 -0700 Subject: [PATCH 22/25] Reverted changes unrelated to this PR; will readd to master --- .vscode/extensions.json | 10 ---------- .vscode/settings.json | 15 --------------- 2 files changed, 25 deletions(-) delete mode 100644 .vscode/extensions.json diff --git a/.vscode/extensions.json b/.vscode/extensions.json deleted file mode 100644 index c27bc02e3..000000000 --- a/.vscode/extensions.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "recommendations": [ - "xaver.clang-format", - "streetsidesoftware.code-spell-checker", - "golang.go", - "davidanson.vscode-markdownlint", - "zxh404.vscode-proto3", - "github.vscode-pull-request-github", - ] -} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 4ea01e578..0b262dc24 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,19 +1,4 @@ { - "clang-format.style": "{ IndentWidth: 4, BasedOnStyle: google, AlignConsecutiveAssignments: true, ColumnLimit: 120 }", - "editor.formatOnSave": true, - "protoc": { - "compile_on_save": false, - "options": [ - "--proto_path=${workspaceRoot}/", - "--proto_path=/Users/meling/go/pkg/mod/github.com/relab/gorums@v0.7.1-0.20220307181651-94a8af8e467c", - // "--proto_path=$(shell go list -m -f {{.Dir}} github.com/relab/gorums)", - // "--proto_path=`go list -m -f {{.Dir}} github.com/relab/gorums`", - ] - }, - "go.lintTool": "golangci-lint", - "go.lintFlags": [ - "--fast", - ], "go.testFlags": [ "-count=1" ], From d626f0199a28d7bc762b045b4b3b447527ec096f Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Wed, 5 Jul 2023 16:31:37 -0700 Subject: [PATCH 23/25] Added newline --- .vscode/settings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 0b262dc24..6b141c503 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,4 +3,5 @@ "-count=1" ], "cSpell.enabled": true -} \ No newline at end of file +} + From 945623766e8fad54b9d548aeb8cacd3a2ee1297d Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Wed, 5 Jul 2023 17:26:03 -0700 Subject: [PATCH 24/25] Fixed compile error in outputErrors() --- fuzz/error.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fuzz/error.go b/fuzz/error.go index 3aa3b3da8..8798eee00 100644 --- a/fuzz/error.go +++ b/fuzz/error.go @@ -62,10 +62,8 @@ func (ei *errorInfo) init() { } func (ei *errorInfo) outputErrors(t *testing.T) { - ei.outputInfo(t) - - for panicInfo := range ei.panics { + for _, panicInfo := range ei.panics { t.Error(panicInfo.Err) } } From b9bd45f1b8842b6f057368b7b4abce400f68e8a0 Mon Sep 17 00:00:00 2001 From: Hein Meling Date: Fri, 11 Aug 2023 14:57:58 -0700 Subject: [PATCH 25/25] Recompile proto files --- internal/proto/hotstuffpb/hotstuff.pb.go | 6 ++++-- internal/proto/hotstuffpb/hotstuff_gorums.pb.go | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/internal/proto/hotstuffpb/hotstuff.pb.go b/internal/proto/hotstuffpb/hotstuff.pb.go index e1226cab4..f452cf838 100644 --- a/internal/proto/hotstuffpb/hotstuff.pb.go +++ b/internal/proto/hotstuffpb/hotstuff.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.21.4 +// protoc-gen-go v1.28.0 +// protoc v4.23.4 // source: internal/proto/hotstuffpb/hotstuff.proto package hotstuffpb @@ -319,6 +319,7 @@ type Signature struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Sig: + // // *Signature_ECDSASig // *Signature_BLS12Sig Sig isSignature_Sig `protobuf_oneof:"Sig"` @@ -556,6 +557,7 @@ type QuorumSignature struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Sig: + // // *QuorumSignature_ECDSASigs // *QuorumSignature_BLS12Sig Sig isQuorumSignature_Sig `protobuf_oneof:"Sig"` diff --git a/internal/proto/hotstuffpb/hotstuff_gorums.pb.go b/internal/proto/hotstuffpb/hotstuff_gorums.pb.go index fc82a8272..1e46a1c9c 100644 --- a/internal/proto/hotstuffpb/hotstuff_gorums.pb.go +++ b/internal/proto/hotstuffpb/hotstuff_gorums.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.7.0-devel -// protoc v3.21.4 +// protoc v4.23.4 // source: internal/proto/hotstuffpb/hotstuff.proto package hotstuffpb @@ -33,8 +33,9 @@ type Configuration struct { // ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. // // This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) +// +// cfg1, err := mgr.NewConfiguration(qspec1, opts...) +// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) *Configuration { // return an error if the QuorumSpec interface is not empty and no implementation was provided. var test interface{} = struct{}{}