From 75379b695e9af007f1cb101e3023bab532eda3ee Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Thu, 3 Jul 2025 16:00:28 +0000 Subject: [PATCH 1/2] Example binary showing log and map being co-hosted This is a demo application that emulates a package repository. Each entry in the log has a {module, version, hash}, and the map simply enumerates all indices where a given module appears. This uses Tessera POSIX implementation for the log, and hooks up the LogReader for that as an InputLog to the Verifiable Index. This starts a web server that hosts a form allowing a package name to be input, and the list of indices are returned. Paths are: - /vindex/ : the verifiable index - /log/ : the log files, served via HTTP too This will make it easier to prototype a demo client that will call into this log/map combo. TODO (in a future PR): - Return inclusion proofs on vindex lookup - Implement an output log --- go.mod | 17 +- go.sum | 99 ++++++- vindex/cmd/logandmap/main.go | 331 +++++++++++++++++++++++ vindex/cmd/logandmap/templates/form.html | 29 ++ vindex/cmd/logandmap/web.go | 90 ++++++ 5 files changed, 562 insertions(+), 4 deletions(-) create mode 100644 vindex/cmd/logandmap/main.go create mode 100644 vindex/cmd/logandmap/templates/form.html create mode 100644 vindex/cmd/logandmap/web.go diff --git a/go.mod b/go.mod index 1f61c2c..93115e0 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,25 @@ go 1.24.1 require ( filippo.io/torchwood v0.5.1-0.20250605130057-fa65d721a6ce + github.com/gorilla/mux v1.8.1 github.com/transparency-dev/formats v0.0.0-20250616090723-6ce2fd29df16 + github.com/transparency-dev/tessera v0.2.1-0.20250702155531-10291081ca03 + golang.org/x/mod v0.25.0 golang.org/x/sync v0.16.0 k8s.io/klog/v2 v2.130.1 ) require ( - github.com/go-logr/logr v1.4.1 // indirect - golang.org/x/mod v0.25.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.2 // indirect + github.com/globocom/go-buffer v1.2.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/transparency-dev/merkle v0.0.2 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect ) diff --git a/go.sum b/go.sum index f935f01..04f4e75 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,112 @@ filippo.io/torchwood v0.5.1-0.20250605130057-fa65d721a6ce h1:8bVOhkZ6uQn3qCmrz/F7Rz0epygH9KO3dgbS9g/zclk= filippo.io/torchwood v0.5.1-0.20250605130057-fa65d721a6ce/go.mod h1:n82QaQC2EzvctEMkM1KAahZaAcQ2uZoCdn051sUXOQY= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= +github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/globocom/go-buffer v1.2.2 h1:ICgtlUe5GIYIZFdAVj57+5WYBR4DA56cX+PYZDhGDwc= +github.com/globocom/go-buffer v1.2.2/go.mod h1:kY1ALQS0ChiiThmWhsFoT5CYSiuad0t3keIew5LsWdM= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/transparency-dev/formats v0.0.0-20250616090723-6ce2fd29df16 h1:4yn2lO94tSuoT8fPm4NJZL5cRMHKOpGwmH/KM9f/ULI= github.com/transparency-dev/formats v0.0.0-20250616090723-6ce2fd29df16/go.mod h1:v+kgcd91U14WBv5EshoX1nooq4SIZTZzYpQDBq6N55U= +github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= +github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= +github.com/transparency-dev/tessera v0.2.1-0.20250702155531-10291081ca03 h1:80Kyf8I5HXuIMYQHf6gfI+AxlVUfvXm/CbnbNoQEkZ4= +github.com/transparency-dev/tessera v0.2.1-0.20250702155531-10291081ca03/go.mod h1:kj8Kq1OpgubsLKBeLqGpk5MGLDvNjtyv40+e6dEW850= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= +golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= diff --git a/vindex/cmd/logandmap/main.go b/vindex/cmd/logandmap/main.go new file mode 100644 index 0000000..eb25bfd --- /dev/null +++ b/vindex/cmd/logandmap/main.go @@ -0,0 +1,331 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// logandmap is a binary that serves as a demo of how to run a log and a map in the +// same process. +// The log is a Tessera POSIX log, and the map is an in-memory verifiable index. +// A web server is hosted that allows lookups in the map to be performed. +// The log is updated periodically with entries of type LogEntry, and the map keys +// each of the module names from that struct to each of the indices in the log where +// an entry for that module is stored. +package main + +import ( + "context" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "flag" + "fmt" + "iter" + "math/rand" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/gorilla/mux" + "github.com/transparency-dev/formats/log" + "github.com/transparency-dev/incubator/vindex" + "github.com/transparency-dev/tessera" + "github.com/transparency-dev/tessera/api" + "github.com/transparency-dev/tessera/client" + "github.com/transparency-dev/tessera/storage/posix" + "golang.org/x/mod/sumdb/note" + "k8s.io/klog/v2" +) + +var ( + posixLogDir = flag.String("posix_log_dir", "", "Root directory in which to store the log for the POSIX backend") + privKeyFile = flag.String("private_key", "", "Location of private key file. If unset, uses the contents of the LOG_PRIVATE_KEY environment variable.") + walPath = flag.String("walPath", "", "Path to use for the Write Ahead Log. If empty, a temporary file will be used.") + addr = flag.String("addr", ":8088", "Address to set up HTTP server listening on") +) + +func main() { + flag.Parse() + klog.InitFlags(nil) + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if err := run(ctx); err != nil { + klog.Exitf("run failed: %v", err) + } +} + +type LogEntry struct { + Module string `json:"module"` + Version string `json:"version"` + Hash []byte `json:"hash"` +} + +func run(ctx context.Context) error { + if *posixLogDir == "" { + return errors.New("posix_log_dir must be set") + } + + // Gather the info needed for reading/writing checkpoints + s, v := getSignerVerifierOrDie() + + // Set up a Tessera POSIX log + driver, err := posix.New(ctx, *posixLogDir) + if err != nil { + return fmt.Errorf("failed to create new log: %v", err) + } + + // Get a Tessera appender + appender, shutdown, reader, err := tessera.NewAppender(ctx, driver, tessera.NewAppendOptions(). + WithCheckpointSigner(s). + WithBatching(256, time.Second)) + if err != nil { + return fmt.Errorf("failed to get appender: %v", err) + } + defer func() { + _ = shutdown(ctx) + }() + + // Create the verifiable index connected to the LogReader. + inputLog := logReaderSource{ + r: reader, + } + logCpParseFn := func(cpRaw []byte) (*log.Checkpoint, error) { + // No witnesses required yet + cp, _, _, err := log.ParseCheckpoint(cpRaw, v.Name(), v) + return cp, err + } + vi, err := vindex.NewVerifiableIndex(ctx, inputLog, logCpParseFn, mapFnFromFlags(), walPathFromFlags()) + if err != nil { + return fmt.Errorf("failed to create vindex: %v", err) + } + + // Submits new entries to the log in the background. + go submitEntries(ctx, appender) + + // Keeps the map synced with the latest published log state. + go maintainMap(ctx, vi) + + // Run a web server to handle queries over the verifiable index. + go runWebServer(vi) + <-ctx.Done() + return nil +} + +// logReaderSource adapts a tessera.LogReader to a vindex.InputLog. +type logReaderSource struct { + r tessera.LogReader +} + +func (s logReaderSource) GetCheckpoint(ctx context.Context) (checkpoint []byte, err error) { + return s.r.ReadCheckpoint(ctx) +} + +func (s logReaderSource) StreamLeaves(ctx context.Context, start, end uint64) iter.Seq2[[]byte, error] { + bi := client.EntryBundles(ctx, 2, s.r.IntegratedSize, s.r.ReadEntryBundle, start, end-start) + unbundleFn := func(bundle []byte) ([][]byte, error) { + eb := &api.EntryBundle{} + if err := eb.UnmarshalText(bundle); err != nil { + return nil, err + } + return eb.Entries, nil + } + + return func(yield func([]byte, error) bool) { + // Unwrap the client.Entry type to return an iterator of []byte only. + for entry, err := range client.Entries(bi, unbundleFn) { + if err != nil { + if !yield(nil, err) { + return + } + continue + } + if !yield(entry.Entry, nil) { + return + } + } + } +} + +// maintainMap reads entries from the log and sync them to the vindex. +func maintainMap(ctx context.Context, vi *vindex.VerifiableIndex) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := vi.Update(ctx); err != nil { + klog.Warning(err) + } + } + } +} + +// submitEntries continually creates new log entries and submits them to the log. +// Entries should be json-encoded LogEntrys. +// The module should be randomly pulled from a list of [foo, bar, baz, splat] +// The version should be the current timestamp, as a string +// The hash should be the sha256 of the module+version +func submitEntries(ctx context.Context, appender *tessera.Appender) { + modules := []string{"foo", "bar", "baz", "splat"} + r := rand.New(rand.NewSource(time.Now().UnixNano())) + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + klog.Info("Context cancelled, stopping log appender") + return + case <-ticker.C: + module := modules[r.Intn(len(modules))] + version := time.Now().Format(time.RFC3339Nano) + h := sha256.Sum256([]byte(module + version)) + entry := LogEntry{ + Module: module, + Version: version, + Hash: h[:], + } + data, err := json.Marshal(entry) + if err != nil { + klog.Errorf("Failed to marshal log entry: %v", err) + continue + } + if idx, err := appender.Add(ctx, tessera.NewEntry(data))(); err != nil { + klog.Errorf("Failed to append to log: %v", err) + } else { + klog.Infof("Appended entry for %s@%s at index %d", module, version, idx.Index) + } + } + } +} + +func runWebServer(vi *vindex.VerifiableIndex) { + web := NewServer(func(s string) string { + idxes := vi.Lookup(s) + return fmt.Sprintf("Indices in log: %v", idxes) + }) + + fs := http.FileServer(http.Dir(*posixLogDir)) + r := mux.NewRouter() + r.PathPrefix("/log/").Handler(http.StripPrefix("/log/", fs)) + web.registerHandlers(r) + hServer := &http.Server{ + Addr: *addr, + Handler: r, + } + go func() { + _ = hServer.ListenAndServe() + }() + klog.Infof("Started HTTP server listening on %s", *addr) +} + +// Read log private key from file or environment variable and generate the +// note Signer and Verifier pair for it. +func getSignerVerifierOrDie() (note.Signer, note.Verifier) { + var privKey string + var err error + if len(*privKeyFile) > 0 { + privKey, err = getKeyFile(*privKeyFile) + if err != nil { + klog.Exitf("Unable to get private key: %v", err) + } + } else { + privKey = os.Getenv("LOG_PRIVATE_KEY") + if len(privKey) == 0 { + klog.Exit("Supply private key file path using --private_key or set LOG_PRIVATE_KEY environment variable") + } + } + s, v, err := signerVerifierFromSkey(privKey) + if err != nil { + klog.Exitf("Failed to get signer/verifier: %v", err) + } + return s, v +} + +func signerVerifierFromSkey(skey string) (note.Signer, note.Verifier, error) { + const algEd25519 = 1 + s, err := note.NewSigner(skey) + if err != nil { + return nil, nil, err + } + _, skey, _ = strings.Cut(skey, "+") + _, skey, _ = strings.Cut(skey, "+") + _, skey, _ = strings.Cut(skey, "+") + _, key64, _ := strings.Cut(skey, "+") + key, err := base64.StdEncoding.DecodeString(key64) + if err != nil { + return nil, nil, fmt.Errorf("failed to decode base64: %v", err) + } + + alg, key := key[0], key[1:] + switch alg { + default: + return nil, nil, errors.New("unsupported algorithm") + + case algEd25519: + if len(key) != ed25519.SeedSize { + return nil, nil, fmt.Errorf("expected key seed of size %d but got %d", ed25519.SeedSize, len(key)) + } + key := ed25519.NewKeyFromSeed(key) + publicKey := key.Public().(ed25519.PublicKey) + vkey, err := note.NewEd25519VerifierKey(s.Name(), publicKey) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate verifier from key: %v", err) + + } + v, err := note.NewVerifier(vkey) + if err != nil { + return nil, nil, fmt.Errorf("failed to create verifier from vkey: %v", err) + } + return s, v, err + } +} + +func getKeyFile(path string) (string, error) { + k, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("failed to read key file: %w", err) + } + return string(k), nil +} + +func mapFnFromFlags() vindex.MapFn { + mapFn := func(data []byte) [][32]byte { + var entry LogEntry + if err := json.Unmarshal(data, &entry); err != nil { + panic(fmt.Errorf("failed to unmarshal entry: %v", err)) + } + + return [][32]byte{sha256.Sum256([]byte(entry.Module))} + } + return mapFn +} + +func walPathFromFlags() string { + if len(*walPath) > 0 { + return *walPath + } + f, err := os.CreateTemp("", "walPath") + if err != nil { + klog.Exitf("Failed to create temporary path for WAL: %s", err) + } + klog.Infof("Created temporary WAL at %s", f.Name()) + return f.Name() +} diff --git a/vindex/cmd/logandmap/templates/form.html b/vindex/cmd/logandmap/templates/form.html new file mode 100644 index 0000000..e6c2afd --- /dev/null +++ b/vindex/cmd/logandmap/templates/form.html @@ -0,0 +1,29 @@ + + + + + + Verifiable Index Lookup + + + +
+

Verifiable Index Lookup

+

{{.Message}}

+
+ + +
+ +
+
+ + diff --git a/vindex/cmd/logandmap/web.go b/vindex/cmd/logandmap/web.go new file mode 100644 index 0000000..148cfe8 --- /dev/null +++ b/vindex/cmd/logandmap/web.go @@ -0,0 +1,90 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + _ "embed" + "fmt" + "html/template" + "net/http" + + "github.com/gorilla/mux" + "k8s.io/klog/v2" +) + +// Define a struct to hold any data we might pass to the template +type FormData struct { + Message string +} + +var ( + //go:embed templates/form.html + templateStr string + tmpl = template.Must(template.New("form").Parse(templateStr)) +) + +func NewServer(lookup func(string) string) Server { + return Server{ + lookup: lookup, + } +} + +type Server struct { + lookup func(string) string +} + +// serveForm handles GET requests to the root path ("/") +// It renders the HTML form. +func (s Server) serveForm(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + // Render the form template with no initial data + err := tmpl.Execute(w, FormData{Message: "Enter your string below:"}) + if err != nil { + klog.Warningf("Error executing template: %v", err) + http.Error(w, "Internal server error", http.StatusInternalServerError) + } +} + +// handleLookup handles POST requests for looking up map entries. +// It parses the form data and responds. +func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + err := r.ParseForm() + if err != nil { + http.Error(w, fmt.Sprintf("Error parsing form: %v", err), http.StatusBadRequest) + return + } + inputString := r.FormValue("inputString") + + klog.V(2).Infof("Received string from form: '%s'", inputString) + + responseMessage := s.lookup(inputString) + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, responseMessage) +} + +func (s Server) registerHandlers(r *mux.Router) { + r.HandleFunc("/vindex/", s.serveForm).Methods("GET") + r.HandleFunc("/vindex/lookup", s.handleLookup).Methods("POST") +} From c2afe412a17cfeabe006cfb462435b89a15d197c Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Wed, 16 Jul 2025 13:43:47 +0000 Subject: [PATCH 2/2] Review comments --- vindex/cmd/logandmap/main.go | 20 ++++++++++++-------- vindex/cmd/logandmap/web.go | 3 +-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/vindex/cmd/logandmap/main.go b/vindex/cmd/logandmap/main.go index eb25bfd..21af84a 100644 --- a/vindex/cmd/logandmap/main.go +++ b/vindex/cmd/logandmap/main.go @@ -54,7 +54,7 @@ var ( posixLogDir = flag.String("posix_log_dir", "", "Root directory in which to store the log for the POSIX backend") privKeyFile = flag.String("private_key", "", "Location of private key file. If unset, uses the contents of the LOG_PRIVATE_KEY environment variable.") walPath = flag.String("walPath", "", "Path to use for the Write Ahead Log. If empty, a temporary file will be used.") - addr = flag.String("addr", ":8088", "Address to set up HTTP server listening on") + listen = flag.String("listen", ":8088", "Address to set up HTTP server listening on") ) func main() { @@ -64,7 +64,7 @@ func main() { defer cancel() if err := run(ctx); err != nil { - klog.Exitf("run failed: %v", err) + klog.Exitf("Run failed: %v", err) } } @@ -178,10 +178,9 @@ func maintainMap(ctx context.Context, vi *vindex.VerifiableIndex) { } // submitEntries continually creates new log entries and submits them to the log. -// Entries should be json-encoded LogEntrys. -// The module should be randomly pulled from a list of [foo, bar, baz, splat] -// The version should be the current timestamp, as a string -// The hash should be the sha256 of the module+version +// Entries are json-encoded LogEntry structs. The module are randomly pulled from a +// list of [foo, bar, baz, splat]. The version is the current timestamp, as a string. +// The hash is set to the sha256 of the module+version. func submitEntries(ctx context.Context, appender *tessera.Appender) { modules := []string{"foo", "bar", "baz", "splat"} r := rand.New(rand.NewSource(time.Now().UnixNano())) @@ -227,13 +226,13 @@ func runWebServer(vi *vindex.VerifiableIndex) { r.PathPrefix("/log/").Handler(http.StripPrefix("/log/", fs)) web.registerHandlers(r) hServer := &http.Server{ - Addr: *addr, + Addr: *listen, Handler: r, } go func() { _ = hServer.ListenAndServe() }() - klog.Infof("Started HTTP server listening on %s", *addr) + klog.Infof("Started HTTP server listening on %s", *listen) } // Read log private key from file or environment variable and generate the @@ -259,6 +258,7 @@ func getSignerVerifierOrDie() (note.Signer, note.Verifier) { return s, v } +// TODO(mhutchinson): move this into t-dev/formats. func signerVerifierFromSkey(skey string) (note.Signer, note.Verifier, error) { const algEd25519 = 1 s, err := note.NewSigner(skey) @@ -313,6 +313,10 @@ func mapFnFromFlags() vindex.MapFn { panic(fmt.Errorf("failed to unmarshal entry: %v", err)) } + // This returns a key which is simply the hash of the module name. + // This could be changed to return something more complex, e.g. include + // a static prefix of "module=", which would allow the same map to host + // multiple queries in parallel. return [][32]byte{sha256.Sum256([]byte(entry.Module))} } return mapFn diff --git a/vindex/cmd/logandmap/web.go b/vindex/cmd/logandmap/web.go index 148cfe8..fb75fb3 100644 --- a/vindex/cmd/logandmap/web.go +++ b/vindex/cmd/logandmap/web.go @@ -69,8 +69,7 @@ func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) { return } - err := r.ParseForm() - if err != nil { + if err := r.ParseForm(); err != nil { http.Error(w, fmt.Sprintf("Error parsing form: %v", err), http.StatusBadRequest) return }