From ec01581775db3e08aaf25171f63059515ef36a31 Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Wed, 9 Jul 2025 11:31:56 +0000 Subject: [PATCH 1/3] Added provisional API definition and client This replaces the web server that was hosted before, and starts moving things in a direction that is closer to what we are likely to run. The API lookup object is defined, but is largely empty for now. Future PRs will work to fill out the remaining fields in there, and then we can implement full verification logic in the client. Bumped Tessera to latest version. --- go.mod | 7 +- go.sum | 74 +----------- vindex/README.md | 40 +++++-- vindex/api/api.go | 62 ++++++++++ vindex/cmd/client/client.go | 138 +++++++++++++++++++++++ vindex/cmd/logandmap/main.go | 27 +++-- vindex/cmd/logandmap/templates/form.html | 29 ----- vindex/cmd/logandmap/web.go | 70 +++++------- vindex/map.go | 13 +-- 9 files changed, 289 insertions(+), 171 deletions(-) create mode 100644 vindex/api/api.go create mode 100644 vindex/cmd/client/client.go delete mode 100644 vindex/cmd/logandmap/templates/form.html diff --git a/go.mod b/go.mod index 93115e0..9bccebe 100644 --- a/go.mod +++ b/go.mod @@ -6,15 +6,14 @@ require ( filippo.io/torchwood v0.5.1-0.20250605130057-fa65d721a6ce github.com/gorilla/mux v1.8.1 github.com/transparency-dev/formats v0.0.0-20250616090723-6ce2fd29df16 - github.com/transparency-dev/tessera v0.2.1-0.20250702155531-10291081ca03 - golang.org/x/mod v0.25.0 + github.com/transparency-dev/tessera v0.2.1-0.20250722085756-7303218c6614 + golang.org/x/mod v0.26.0 golang.org/x/sync v0.16.0 k8s.io/klog/v2 v2.130.1 ) require ( github.com/cenkalti/backoff/v5 v5.0.2 // indirect - github.com/globocom/go-buffer v1.2.2 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect @@ -23,6 +22,6 @@ require ( go.opentelemetry.io/otel v1.37.0 // indirect go.opentelemetry.io/otel/metric v1.37.0 // indirect go.opentelemetry.io/otel/trace v1.37.0 // indirect - golang.org/x/crypto v0.39.0 // indirect + golang.org/x/crypto v0.40.0 // indirect golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect ) diff --git a/go.sum b/go.sum index 04f4e75..e8c69c2 100644 --- a/go.sum +++ b/go.sum @@ -4,58 +4,29 @@ github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7 github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/globocom/go-buffer v1.2.2 h1:ICgtlUe5GIYIZFdAVj57+5WYBR4DA56cX+PYZDhGDwc= -github.com/globocom/go-buffer v1.2.2/go.mod h1:kY1ALQS0ChiiThmWhsFoT5CYSiuad0t3keIew5LsWdM= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= -github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/transparency-dev/formats v0.0.0-20250616090723-6ce2fd29df16 h1:4yn2lO94tSuoT8fPm4NJZL5cRMHKOpGwmH/KM9f/ULI= github.com/transparency-dev/formats v0.0.0-20250616090723-6ce2fd29df16/go.mod h1:v+kgcd91U14WBv5EshoX1nooq4SIZTZzYpQDBq6N55U= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= -github.com/transparency-dev/tessera v0.2.1-0.20250702155531-10291081ca03 h1:80Kyf8I5HXuIMYQHf6gfI+AxlVUfvXm/CbnbNoQEkZ4= -github.com/transparency-dev/tessera v0.2.1-0.20250702155531-10291081ca03/go.mod h1:kj8Kq1OpgubsLKBeLqGpk5MGLDvNjtyv40+e6dEW850= +github.com/transparency-dev/tessera v0.2.1-0.20250722085756-7303218c6614 h1:EfxzWae/zdnfVA44jFIKohvd2JWlcanRzRYb1A/uiDU= +github.com/transparency-dev/tessera v0.2.1-0.20250722085756-7303218c6614/go.mod h1:ilpKqGrwDD/6uop5nDj/X60o0qt33GK1uInjIcfZTP0= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= @@ -64,47 +35,14 @@ go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/Wgbsd go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= -golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= -golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= diff --git a/vindex/README.md b/vindex/README.md index 1b5e263..302f809 100644 --- a/vindex/README.md +++ b/vindex/README.md @@ -7,6 +7,9 @@ This idea has been distilled from years of experiments with maps, and a pressing This experiment should be considered a 20% project for the time being and isn't on the near-term official roadmap for transparency.dev. Discussions are welcome, please join us on [Transparency-Dev Slack](https://transparency.dev/slack/). +[tlog-tiles]: https://c2sp.org/tlog-tiles +[Tessera]: https://github.com/transparency-dev/tessera + ## Overview ### The Problem: Verifiability vs. Efficiency @@ -182,19 +185,34 @@ Values are an ordered list of indices. ## Status -There is a basic end-to-end application written that currently only supports SumDB. -This application: - - Reads a local [SumDB clone](https://github.com/google/trillian-examples/tree/master/clone/cmd/sumdbclone) - - Builds a map, in the format described above - - Serves a Lookup API via HTTP +There is a basic end-to-end application written that supports SumDB in [trillian-examples](https://github.com/google/trillian-examples/tree/master/experimental/vindex/cmd). + +In this repository, there is a demo of running a [tlog-tiles][] log using [Tessera][], and keeping the contents of that log synced to a map. +Below are instructions for running this demo with sample key material: + +```shell +LOG_PRIVATE_KEY=PRIVATE+KEY+logandmap+38581672+AXJ0FKWOcO2ch6WC8kP705Ed3Gxu7pVtZLhfHAQwp+FE; go run ./vindex/cmd/logandmap --input_log_dir ~/logandmap/inputlog/ --walPath ~/logandmap/map.wal +``` + +Running the above will run a web server hosting the following URLs: + - `/inputlog/` - the [tlog-tiles][] + - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go) + - `/outputlog/` - TODO(mhutchinson): this is where the output log will be hosted -Running it: - 1. Have a fully mirrored SumDB using `sumdbclone` (the easiest way is to use `docker compose up`, and wait) - 1. Run the verifiable index binary: `go run ./experimental/vindex/cmd --logDSN="sumdb:letmein@tcp(127.0.0.1:33006)/sumdb" --walPath ~/sumdb.wal` +The input log has entries for packages in the set {`foo`, `bar`, `baz`, `splat`}. +To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above): -Providing the above works, you will have a web server hosting a form at http://localhost:8088. -This form takes a package name for SumDB (e.g. `github.com/transparency-dev/tessera`), and outputs all indices in the SumDB log corresponding to that package. -You will also have a WAL file at `~/sumdb.wal`, which will make future boots faster. +```shell +go run github.com/mhutchinson/woodpecker@main --custom_log_type=tiles --custom_log_url=http://localhost:8088/inputlog --custom_log_vkey=logandmap+38581672+Ab/PCr1eCclRPRMBqw/r5An1xO71MCnImLiospEq6b4l +``` + +Use left/right cursor to browse, and `q` to quit. + +This log is processed into a verifiable map which can be looked up using the following command: + +```shell +go run ./vindex/cmd/client --base_url http://localhost:8088/vindex/ --key=foo +``` ## Milestones diff --git a/vindex/api/api.go b/vindex/api/api.go new file mode 100644 index 0000000..294f945 --- /dev/null +++ b/vindex/api/api.go @@ -0,0 +1,62 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// api contains the API definitions for the prototype VIndex API. +package api + +import "crypto/sha256" + +const ( + // PathLookup defines the path from the vindex base URL where the lookup + // operation will be served. This takes a single key as a GET request + // parameter, and returns a marshalled LookupResponse. + PathLookup = "/lookup/" +) + +// LookupResponse describes the result from a lookup operation. +type LookupResponse struct { + // These values represent the proof that the state of the index is committed + // to in the output log. These fields are a checkpoint, the encoded leaf in + // this log (which can be parsed as an OutputLogLeaf), and a proof that the leaf + // is committeed to by the checkpoint. + // + // TODO(mhutchinson): Revisit these fields when implementing the output log. + // A more modern approach is to return only an index into the output log, and then + // have the client look up the latest checkpoint, leaf value, and inclusion proof. + // This puts more work on the client, but saves work for the server and provides + // more flexibility. + OutputLogCP []byte `json:"output_log_cp"` + OutputLogLeaf []byte `json:"output_log_leaf"` + OutputLogProof [][sha256.Size]byte `json:"output_log_proof"` + + // These values represent the lookup operation in the index at the root hash + // committed to by OutputLogLeaf. The values contain all indices for the given + // key, and the proof binds these values at this key at the index root hash. + IndexKey [sha256.Size]byte `json:"index_key"` + IndexValue []uint64 `json:"index_value"` + IndexProof [][sha256.Size]byte `json:"index_proof"` +} + +// OutputLogLeaf describes a leaf in the output log. +// +// This leaf is a statement that commits to the result of generating a map from a +// given input log size. To do this, it binds the Input Log state (in the form of +// a checkpoint), to a single hash, which is the Merkle Tree root hash for the index. +type OutputLogLeaf struct { + // InputLogCP is the checkpoint from the input log that commits to + // the state of the log that the index was derived from. + InputLogCP []byte `json:"input_log_cp"` + // IndexRoot is the root hash of the Merkle Tree for the verifiable index. + IndexRoot [sha256.Size]byte `json:"index_root"` +} diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go new file mode 100644 index 0000000..9da9a01 --- /dev/null +++ b/vindex/cmd/client/client.go @@ -0,0 +1,138 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// client defines a binary that looks up a key in a vindex, verifying all +// proofs. Optionally, it can derefence these indices and fetch verified +// leaf entries from the input log. +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "net/url" + + "github.com/transparency-dev/incubator/vindex/api" + "k8s.io/klog/v2" +) + +var ( + baseURL = flag.String("base_url", "", "The base URL of the vindex server.") + key = flag.String("key", "", "The key to look up in the vindex.") +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + if err := run(context.Background()); err != nil { + klog.Exitf("run failed: %v", err) + } +} + +func run(ctx context.Context) error { + c := newVIndexClientFromFlags() + + if *key == "" { + klog.Exit("key flag must be provided") + } + + resp, err := c.Lookup(ctx, *key) + if err != nil { + return fmt.Errorf("lookup for %q failed: %v", *key, err) + } + + // For now, pretty print the JSON response + jsonResp, err := json.MarshalIndent(resp, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal response: %v", err) + } + fmt.Println(string(jsonResp)) + + // This needs to verify the proofs in the response + // We can't verify inclusion with the info we currently have + // We at least need the index of the leaf returned. We could add + // that to the response object, but this is kinda rfc6962. + // What would it look like if the output log is tiled? What should + // the lookup response contain? Can it contain less, and thus put + // more on the client as per tlog-tiles? If so, what stops clients + // requesting old views of the index based on non-recent leaves from + // the output log? + // What if we flip this all around, and the OutputLog part of the response + // only returns an index into the output log, and the client has to look up + // that leaf, checkpoint, and generate inclusion proof? + + // proof.VerifyInclusion() + + return nil +} + +func newVIndexClientFromFlags() VIndexClient { + if *baseURL == "" { + klog.Exit("base_url flag must be provided") + } + u, err := url.Parse(*baseURL) + if err != nil { + klog.Exitf("failed to parse URL: %v", err) + } + lookupURL := u.JoinPath(api.PathLookup) + return VIndexClient{ + lookupURL: lookupURL, + } +} + +type VIndexClient struct { + lookupURL *url.URL +} + +func (c VIndexClient) Lookup(ctx context.Context, key string) (api.LookupResponse, error) { + var lookupResp api.LookupResponse + + // For now, keys are stored under the hash of the key + kh := sha256.Sum256([]byte(key)) + u := c.lookupURL.JoinPath(hex.EncodeToString(kh[:])) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return lookupResp, fmt.Errorf("failed to create request: %v", err) + } + + klog.Infof("Making request to %q", u.String()) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return lookupResp, fmt.Errorf("failed to get URL %q: %v", u, err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + return lookupResp, fmt.Errorf("got non-200 status code: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return lookupResp, fmt.Errorf("failed to read response body: %v", err) + } + + if err := json.Unmarshal(body, &lookupResp); err != nil { + return lookupResp, fmt.Errorf("failed to unmarshal response: %v", err) + } + return lookupResp, nil +} diff --git a/vindex/cmd/logandmap/main.go b/vindex/cmd/logandmap/main.go index 21af84a..012144c 100644 --- a/vindex/cmd/logandmap/main.go +++ b/vindex/cmd/logandmap/main.go @@ -51,15 +51,16 @@ import ( ) var ( - posixLogDir = flag.String("posix_log_dir", "", "Root directory in which to store the log for the POSIX backend") privKeyFile = flag.String("private_key", "", "Location of private key file. If unset, uses the contents of the LOG_PRIVATE_KEY environment variable.") + inputLogDir = flag.String("input_log_dir", "", "Root directory in which to store the log for the POSIX-based Input Log") walPath = flag.String("walPath", "", "Path to use for the Write Ahead Log. If empty, a temporary file will be used.") listen = flag.String("listen", ":8088", "Address to set up HTTP server listening on") ) func main() { - flag.Parse() klog.InitFlags(nil) + flag.Parse() + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancel() @@ -75,15 +76,15 @@ type LogEntry struct { } func run(ctx context.Context) error { - if *posixLogDir == "" { - return errors.New("posix_log_dir must be set") + if *inputLogDir == "" { + return errors.New("input_log_dir must be set") } // Gather the info needed for reading/writing checkpoints s, v := getSignerVerifierOrDie() // Set up a Tessera POSIX log - driver, err := posix.New(ctx, *posixLogDir) + driver, err := posix.New(ctx, posix.Config{Path: *inputLogDir}) if err != nil { return fmt.Errorf("failed to create new log: %v", err) } @@ -91,6 +92,7 @@ func run(ctx context.Context) error { // Get a Tessera appender appender, shutdown, reader, err := tessera.NewAppender(ctx, driver, tessera.NewAppendOptions(). WithCheckpointSigner(s). + WithCheckpointInterval(10*time.Second). WithBatching(256, time.Second)) if err != nil { return fmt.Errorf("failed to get appender: %v", err) @@ -209,21 +211,24 @@ func submitEntries(ctx context.Context, appender *tessera.Appender) { if idx, err := appender.Add(ctx, tessera.NewEntry(data))(); err != nil { klog.Errorf("Failed to append to log: %v", err) } else { - klog.Infof("Appended entry for %s@%s at index %d", module, version, idx.Index) + klog.V(2).Infof("Appended entry for %s@%s at index %d", module, version, idx.Index) } } } } func runWebServer(vi *vindex.VerifiableIndex) { - web := NewServer(func(s string) string { - idxes := vi.Lookup(s) - return fmt.Sprintf("Indices in log: %v", idxes) + web := NewServer(func(h [sha256.Size]byte) ([]uint64, error) { + idxes, size := vi.Lookup(h) + if size == 0 { + return nil, errors.New("index not populated") + } + return idxes, nil }) - fs := http.FileServer(http.Dir(*posixLogDir)) + ilfs := http.FileServer(http.Dir(*inputLogDir)) r := mux.NewRouter() - r.PathPrefix("/log/").Handler(http.StripPrefix("/log/", fs)) + r.PathPrefix("/inputlog/").Handler(http.StripPrefix("/inputlog/", ilfs)) web.registerHandlers(r) hServer := &http.Server{ Addr: *listen, diff --git a/vindex/cmd/logandmap/templates/form.html b/vindex/cmd/logandmap/templates/form.html deleted file mode 100644 index e6c2afd..0000000 --- a/vindex/cmd/logandmap/templates/form.html +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - Verifiable Index Lookup - - - -
-

Verifiable Index Lookup

-

{{.Message}}

-
- - -
- -
-
- - diff --git a/vindex/cmd/logandmap/web.go b/vindex/cmd/logandmap/web.go index fb75fb3..1e4defd 100644 --- a/vindex/cmd/logandmap/web.go +++ b/vindex/cmd/logandmap/web.go @@ -15,75 +15,63 @@ package main import ( + "crypto/sha256" _ "embed" + "encoding/hex" + "encoding/json" "fmt" - "html/template" "net/http" "github.com/gorilla/mux" + "github.com/transparency-dev/incubator/vindex/api" "k8s.io/klog/v2" ) -// Define a struct to hold any data we might pass to the template -type FormData struct { - Message string -} - -var ( - //go:embed templates/form.html - templateStr string - tmpl = template.Must(template.New("form").Parse(templateStr)) -) - -func NewServer(lookup func(string) string) Server { +func NewServer(lookup func([sha256.Size]byte) ([]uint64, error)) Server { return Server{ lookup: lookup, } } type Server struct { - lookup func(string) string + lookup func([sha256.Size]byte) ([]uint64, error) } -// serveForm handles GET requests to the root path ("/") -// It renders the HTML form. -func (s Server) serveForm(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) +// handleLookup handles GET requests for looking up map entries. +func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + hashStr, ok := vars["hash"] + if !ok { + http.Error(w, "hash parameter not found", http.StatusBadRequest) return } - // Render the form template with no initial data - err := tmpl.Execute(w, FormData{Message: "Enter your string below:"}) + h, err := hex.DecodeString(hashStr) if err != nil { - klog.Warningf("Error executing template: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) - } -} - -// handleLookup handles POST requests for looking up map entries. -// It parses the form data and responds. -func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + http.Error(w, fmt.Sprintf("invalid hex hash: %v", err), http.StatusBadRequest) return } - - if err := r.ParseForm(); err != nil { - http.Error(w, fmt.Sprintf("Error parsing form: %v", err), http.StatusBadRequest) + if len(h) != sha256.Size { + http.Error(w, fmt.Sprintf("hash wrong length (decoded %d bytes)", len(h)), http.StatusBadRequest) return } - inputString := r.FormValue("inputString") - klog.V(2).Infof("Received string from form: '%s'", inputString) + klog.V(2).Infof("Received hash from request: '%s'", h) - responseMessage := s.lookup(inputString) - w.Header().Set("Content-Type", "text/plain; charset=utf-8") + idxes, err := s.lookup([sha256.Size]byte(h)) + if err != nil { + http.Error(w, fmt.Sprintf("lookup failed: %v", err), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - _, _ = fmt.Fprint(w, responseMessage) + var resp api.LookupResponse + resp.IndexValue = idxes + if err := json.NewEncoder(w).Encode(resp); err != nil { + klog.Warningf("failed to encode response: %v", err) + } } func (s Server) registerHandlers(r *mux.Router) { - r.HandleFunc("/vindex/", s.serveForm).Methods("GET") - r.HandleFunc("/vindex/lookup", s.handleLookup).Methods("POST") + r.HandleFunc("/vindex/lookup/{hash}", s.handleLookup).Methods("GET") } diff --git a/vindex/map.go b/vindex/map.go index 356edea..bac7cdc 100644 --- a/vindex/map.go +++ b/vindex/map.go @@ -132,28 +132,27 @@ func (b *VerifiableIndex) Close() error { // Lookup returns the values stored for the given key. // TODO(mhutchinson): This needs to return verifiable stuff -func (b *VerifiableIndex) Lookup(key string) (indices []uint64) { +func (b *VerifiableIndex) Lookup(key [sha256.Size]byte) (indices []uint64, size uint64) { // Scope the lock to be as minimal as possible - lookupLocked := func(key string) []uint64 { + lookupLocked := func(key [sha256.Size]byte) []uint64 { b.indexMu.RLock() defer b.indexMu.RUnlock() - kh := sha256.Sum256([]byte(key)) - return b.data[kh] + return b.data[key] } // TODO(mhutchinson): this should come from the latest map root in the (witnessed) output log. // This map root, the witnessed output log checkpoint, and all proofs should also be served here. - size := b.servingSize + size = b.servingSize allIndices := lookupLocked(key) for i, idx := range allIndices { if idx >= size { // If we have indices past the current size we are serving, drop them. // Doing this allows us to update b.data with new indices while still serving from it. - return allIndices[:i] + return allIndices[:i], size } } - return allIndices + return allIndices, size } // Update checks the input log for a new Checkpoint, and ensures that the Verifiable Index From d2e89c2ad4bca73e086712292a3e372b1a56c447 Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Tue, 22 Jul 2025 13:56:01 +0100 Subject: [PATCH 2/3] Apply suggestions from code review Co-authored-by: Al Cutter --- vindex/cmd/logandmap/web.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vindex/cmd/logandmap/web.go b/vindex/cmd/logandmap/web.go index 1e4defd..19f986e 100644 --- a/vindex/cmd/logandmap/web.go +++ b/vindex/cmd/logandmap/web.go @@ -51,7 +51,7 @@ func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) { http.Error(w, fmt.Sprintf("invalid hex hash: %v", err), http.StatusBadRequest) return } - if len(h) != sha256.Size { + if l := len(h); l != sha256.Size { http.Error(w, fmt.Sprintf("hash wrong length (decoded %d bytes)", len(h)), http.StatusBadRequest) return } From 63319219f6026735a52208ff3c329b7542b04c4c Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Tue, 22 Jul 2025 13:00:48 +0000 Subject: [PATCH 3/3] more review stuff --- vindex/README.md | 2 +- vindex/api/api.go | 5 +++-- vindex/cmd/client/client.go | 15 ++++++++------- vindex/cmd/logandmap/web.go | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/vindex/README.md b/vindex/README.md index 302f809..cafa73d 100644 --- a/vindex/README.md +++ b/vindex/README.md @@ -211,7 +211,7 @@ Use left/right cursor to browse, and `q` to quit. This log is processed into a verifiable map which can be looked up using the following command: ```shell -go run ./vindex/cmd/client --base_url http://localhost:8088/vindex/ --key=foo +go run ./vindex/cmd/client --base_url http://localhost:8088/vindex/ --lookup=foo ``` ## Milestones diff --git a/vindex/api/api.go b/vindex/api/api.go index 294f945..c905bdb 100644 --- a/vindex/api/api.go +++ b/vindex/api/api.go @@ -19,8 +19,9 @@ import "crypto/sha256" const ( // PathLookup defines the path from the vindex base URL where the lookup - // operation will be served. This takes a single key as a GET request - // parameter, and returns a marshalled LookupResponse. + // operation will be served. This should be constructed by appending a + // hex encoded hash to look up, and the server returns a marshalled + // LookupResponse. PathLookup = "/lookup/" ) diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index 9da9a01..b01a036 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -22,6 +22,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "flag" "fmt" "io" @@ -34,7 +35,7 @@ import ( var ( baseURL = flag.String("base_url", "", "The base URL of the vindex server.") - key = flag.String("key", "", "The key to look up in the vindex.") + lookup = flag.String("lookup", "", "The key to look up in the vindex.") ) func main() { @@ -48,13 +49,13 @@ func main() { func run(ctx context.Context) error { c := newVIndexClientFromFlags() - if *key == "" { - klog.Exit("key flag must be provided") + if *lookup == "" { + return errors.New("key flag must be provided") } - resp, err := c.Lookup(ctx, *key) + resp, err := c.Lookup(ctx, *lookup) if err != nil { - return fmt.Errorf("lookup for %q failed: %v", *key, err) + return fmt.Errorf("lookup for %q failed: %v", *lookup, err) } // For now, pretty print the JSON response @@ -64,8 +65,8 @@ func run(ctx context.Context) error { } fmt.Println(string(jsonResp)) - // This needs to verify the proofs in the response - // We can't verify inclusion with the info we currently have + // This needs to verify the proofs in the response. + // We can't verify inclusion with the info we currently have. // We at least need the index of the leaf returned. We could add // that to the response object, but this is kinda rfc6962. // What would it look like if the output log is tiled? What should diff --git a/vindex/cmd/logandmap/web.go b/vindex/cmd/logandmap/web.go index 19f986e..c0efd92 100644 --- a/vindex/cmd/logandmap/web.go +++ b/vindex/cmd/logandmap/web.go @@ -52,7 +52,7 @@ func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) { return } if l := len(h); l != sha256.Size { - http.Error(w, fmt.Sprintf("hash wrong length (decoded %d bytes)", len(h)), http.StatusBadRequest) + http.Error(w, fmt.Sprintf("hash wrong length (decoded %d bytes)", l), http.StatusBadRequest) return }