diff --git a/go.mod b/go.mod
index 93115e0..9bccebe 100644
--- a/go.mod
+++ b/go.mod
@@ -6,15 +6,14 @@ require (
filippo.io/torchwood v0.5.1-0.20250605130057-fa65d721a6ce
github.com/gorilla/mux v1.8.1
github.com/transparency-dev/formats v0.0.0-20250616090723-6ce2fd29df16
- github.com/transparency-dev/tessera v0.2.1-0.20250702155531-10291081ca03
- golang.org/x/mod v0.25.0
+ github.com/transparency-dev/tessera v0.2.1-0.20250722085756-7303218c6614
+ golang.org/x/mod v0.26.0
golang.org/x/sync v0.16.0
k8s.io/klog/v2 v2.130.1
)
require (
github.com/cenkalti/backoff/v5 v5.0.2 // indirect
- github.com/globocom/go-buffer v1.2.2 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
@@ -23,6 +22,6 @@ require (
go.opentelemetry.io/otel v1.37.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/otel/trace v1.37.0 // indirect
- golang.org/x/crypto v0.39.0 // indirect
+ golang.org/x/crypto v0.40.0 // indirect
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect
)
diff --git a/go.sum b/go.sum
index 04f4e75..e8c69c2 100644
--- a/go.sum
+++ b/go.sum
@@ -4,58 +4,29 @@ github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7
github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
-github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
-github.com/globocom/go-buffer v1.2.2 h1:ICgtlUe5GIYIZFdAVj57+5WYBR4DA56cX+PYZDhGDwc=
-github.com/globocom/go-buffer v1.2.2/go.mod h1:kY1ALQS0ChiiThmWhsFoT5CYSiuad0t3keIew5LsWdM=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
-github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
-github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
-github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/transparency-dev/formats v0.0.0-20250616090723-6ce2fd29df16 h1:4yn2lO94tSuoT8fPm4NJZL5cRMHKOpGwmH/KM9f/ULI=
github.com/transparency-dev/formats v0.0.0-20250616090723-6ce2fd29df16/go.mod h1:v+kgcd91U14WBv5EshoX1nooq4SIZTZzYpQDBq6N55U=
github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4=
github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A=
-github.com/transparency-dev/tessera v0.2.1-0.20250702155531-10291081ca03 h1:80Kyf8I5HXuIMYQHf6gfI+AxlVUfvXm/CbnbNoQEkZ4=
-github.com/transparency-dev/tessera v0.2.1-0.20250702155531-10291081ca03/go.mod h1:kj8Kq1OpgubsLKBeLqGpk5MGLDvNjtyv40+e6dEW850=
+github.com/transparency-dev/tessera v0.2.1-0.20250722085756-7303218c6614 h1:EfxzWae/zdnfVA44jFIKohvd2JWlcanRzRYb1A/uiDU=
+github.com/transparency-dev/tessera v0.2.1-0.20250722085756-7303218c6614/go.mod h1:ilpKqGrwDD/6uop5nDj/X60o0qt33GK1uInjIcfZTP0=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
@@ -64,47 +35,14 @@ go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/Wgbsd
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
-golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
+golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
+golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw=
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
-golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
-golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
-golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg=
+golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ=
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
-golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
diff --git a/vindex/README.md b/vindex/README.md
index 1b5e263..cafa73d 100644
--- a/vindex/README.md
+++ b/vindex/README.md
@@ -7,6 +7,9 @@ This idea has been distilled from years of experiments with maps, and a pressing
This experiment should be considered a 20% project for the time being and isn't on the near-term official roadmap for transparency.dev.
Discussions are welcome, please join us on [Transparency-Dev Slack](https://transparency.dev/slack/).
+[tlog-tiles]: https://c2sp.org/tlog-tiles
+[Tessera]: https://github.com/transparency-dev/tessera
+
## Overview
### The Problem: Verifiability vs. Efficiency
@@ -182,19 +185,34 @@ Values are an ordered list of indices.
## Status
-There is a basic end-to-end application written that currently only supports SumDB.
-This application:
- - Reads a local [SumDB clone](https://github.com/google/trillian-examples/tree/master/clone/cmd/sumdbclone)
- - Builds a map, in the format described above
- - Serves a Lookup API via HTTP
+There is a basic end-to-end application written that supports SumDB in [trillian-examples](https://github.com/google/trillian-examples/tree/master/experimental/vindex/cmd).
+
+In this repository, there is a demo of running a [tlog-tiles][] log using [Tessera][], and keeping the contents of that log synced to a map.
+Below are instructions for running this demo with sample key material:
+
+```shell
+LOG_PRIVATE_KEY=PRIVATE+KEY+logandmap+38581672+AXJ0FKWOcO2ch6WC8kP705Ed3Gxu7pVtZLhfHAQwp+FE; go run ./vindex/cmd/logandmap --input_log_dir ~/logandmap/inputlog/ --walPath ~/logandmap/map.wal
+```
+
+Running the above will run a web server hosting the following URLs:
+ - `/inputlog/` - the [tlog-tiles][]
+ - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go)
+ - `/outputlog/` - TODO(mhutchinson): this is where the output log will be hosted
-Running it:
- 1. Have a fully mirrored SumDB using `sumdbclone` (the easiest way is to use `docker compose up`, and wait)
- 1. Run the verifiable index binary: `go run ./experimental/vindex/cmd --logDSN="sumdb:letmein@tcp(127.0.0.1:33006)/sumdb" --walPath ~/sumdb.wal`
+The input log has entries for packages in the set {`foo`, `bar`, `baz`, `splat`}.
+To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above):
-Providing the above works, you will have a web server hosting a form at http://localhost:8088.
-This form takes a package name for SumDB (e.g. `github.com/transparency-dev/tessera`), and outputs all indices in the SumDB log corresponding to that package.
-You will also have a WAL file at `~/sumdb.wal`, which will make future boots faster.
+```shell
+go run github.com/mhutchinson/woodpecker@main --custom_log_type=tiles --custom_log_url=http://localhost:8088/inputlog --custom_log_vkey=logandmap+38581672+Ab/PCr1eCclRPRMBqw/r5An1xO71MCnImLiospEq6b4l
+```
+
+Use left/right cursor to browse, and `q` to quit.
+
+This log is processed into a verifiable map which can be looked up using the following command:
+
+```shell
+go run ./vindex/cmd/client --base_url http://localhost:8088/vindex/ --lookup=foo
+```
## Milestones
diff --git a/vindex/api/api.go b/vindex/api/api.go
new file mode 100644
index 0000000..c905bdb
--- /dev/null
+++ b/vindex/api/api.go
@@ -0,0 +1,63 @@
+// Copyright 2025 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// api contains the API definitions for the prototype VIndex API.
+package api
+
+import "crypto/sha256"
+
+const (
+ // PathLookup defines the path from the vindex base URL where the lookup
+ // operation will be served. This should be constructed by appending a
+ // hex encoded hash to look up, and the server returns a marshalled
+ // LookupResponse.
+ PathLookup = "/lookup/"
+)
+
+// LookupResponse describes the result from a lookup operation.
+type LookupResponse struct {
+ // These values represent the proof that the state of the index is committed
+ // to in the output log. These fields are a checkpoint, the encoded leaf in
+ // this log (which can be parsed as an OutputLogLeaf), and a proof that the leaf
+ // is committeed to by the checkpoint.
+ //
+ // TODO(mhutchinson): Revisit these fields when implementing the output log.
+ // A more modern approach is to return only an index into the output log, and then
+ // have the client look up the latest checkpoint, leaf value, and inclusion proof.
+ // This puts more work on the client, but saves work for the server and provides
+ // more flexibility.
+ OutputLogCP []byte `json:"output_log_cp"`
+ OutputLogLeaf []byte `json:"output_log_leaf"`
+ OutputLogProof [][sha256.Size]byte `json:"output_log_proof"`
+
+ // These values represent the lookup operation in the index at the root hash
+ // committed to by OutputLogLeaf. The values contain all indices for the given
+ // key, and the proof binds these values at this key at the index root hash.
+ IndexKey [sha256.Size]byte `json:"index_key"`
+ IndexValue []uint64 `json:"index_value"`
+ IndexProof [][sha256.Size]byte `json:"index_proof"`
+}
+
+// OutputLogLeaf describes a leaf in the output log.
+//
+// This leaf is a statement that commits to the result of generating a map from a
+// given input log size. To do this, it binds the Input Log state (in the form of
+// a checkpoint), to a single hash, which is the Merkle Tree root hash for the index.
+type OutputLogLeaf struct {
+ // InputLogCP is the checkpoint from the input log that commits to
+ // the state of the log that the index was derived from.
+ InputLogCP []byte `json:"input_log_cp"`
+ // IndexRoot is the root hash of the Merkle Tree for the verifiable index.
+ IndexRoot [sha256.Size]byte `json:"index_root"`
+}
diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go
new file mode 100644
index 0000000..b01a036
--- /dev/null
+++ b/vindex/cmd/client/client.go
@@ -0,0 +1,139 @@
+// Copyright 2025 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// client defines a binary that looks up a key in a vindex, verifying all
+// proofs. Optionally, it can derefence these indices and fetch verified
+// leaf entries from the input log.
+package main
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+
+ "github.com/transparency-dev/incubator/vindex/api"
+ "k8s.io/klog/v2"
+)
+
+var (
+ baseURL = flag.String("base_url", "", "The base URL of the vindex server.")
+ lookup = flag.String("lookup", "", "The key to look up in the vindex.")
+)
+
+func main() {
+ klog.InitFlags(nil)
+ flag.Parse()
+ if err := run(context.Background()); err != nil {
+ klog.Exitf("run failed: %v", err)
+ }
+}
+
+func run(ctx context.Context) error {
+ c := newVIndexClientFromFlags()
+
+ if *lookup == "" {
+ return errors.New("key flag must be provided")
+ }
+
+ resp, err := c.Lookup(ctx, *lookup)
+ if err != nil {
+ return fmt.Errorf("lookup for %q failed: %v", *lookup, err)
+ }
+
+ // For now, pretty print the JSON response
+ jsonResp, err := json.MarshalIndent(resp, "", " ")
+ if err != nil {
+ return fmt.Errorf("failed to marshal response: %v", err)
+ }
+ fmt.Println(string(jsonResp))
+
+ // This needs to verify the proofs in the response.
+ // We can't verify inclusion with the info we currently have.
+ // We at least need the index of the leaf returned. We could add
+ // that to the response object, but this is kinda rfc6962.
+ // What would it look like if the output log is tiled? What should
+ // the lookup response contain? Can it contain less, and thus put
+ // more on the client as per tlog-tiles? If so, what stops clients
+ // requesting old views of the index based on non-recent leaves from
+ // the output log?
+ // What if we flip this all around, and the OutputLog part of the response
+ // only returns an index into the output log, and the client has to look up
+ // that leaf, checkpoint, and generate inclusion proof?
+
+ // proof.VerifyInclusion()
+
+ return nil
+}
+
+func newVIndexClientFromFlags() VIndexClient {
+ if *baseURL == "" {
+ klog.Exit("base_url flag must be provided")
+ }
+ u, err := url.Parse(*baseURL)
+ if err != nil {
+ klog.Exitf("failed to parse URL: %v", err)
+ }
+ lookupURL := u.JoinPath(api.PathLookup)
+ return VIndexClient{
+ lookupURL: lookupURL,
+ }
+}
+
+type VIndexClient struct {
+ lookupURL *url.URL
+}
+
+func (c VIndexClient) Lookup(ctx context.Context, key string) (api.LookupResponse, error) {
+ var lookupResp api.LookupResponse
+
+ // For now, keys are stored under the hash of the key
+ kh := sha256.Sum256([]byte(key))
+ u := c.lookupURL.JoinPath(hex.EncodeToString(kh[:]))
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+ if err != nil {
+ return lookupResp, fmt.Errorf("failed to create request: %v", err)
+ }
+
+ klog.Infof("Making request to %q", u.String())
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return lookupResp, fmt.Errorf("failed to get URL %q: %v", u, err)
+ }
+ defer func() {
+ _ = resp.Body.Close()
+ }()
+
+ if resp.StatusCode != http.StatusOK {
+ return lookupResp, fmt.Errorf("got non-200 status code: %d", resp.StatusCode)
+ }
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return lookupResp, fmt.Errorf("failed to read response body: %v", err)
+ }
+
+ if err := json.Unmarshal(body, &lookupResp); err != nil {
+ return lookupResp, fmt.Errorf("failed to unmarshal response: %v", err)
+ }
+ return lookupResp, nil
+}
diff --git a/vindex/cmd/logandmap/main.go b/vindex/cmd/logandmap/main.go
index 21af84a..012144c 100644
--- a/vindex/cmd/logandmap/main.go
+++ b/vindex/cmd/logandmap/main.go
@@ -51,15 +51,16 @@ import (
)
var (
- posixLogDir = flag.String("posix_log_dir", "", "Root directory in which to store the log for the POSIX backend")
privKeyFile = flag.String("private_key", "", "Location of private key file. If unset, uses the contents of the LOG_PRIVATE_KEY environment variable.")
+ inputLogDir = flag.String("input_log_dir", "", "Root directory in which to store the log for the POSIX-based Input Log")
walPath = flag.String("walPath", "", "Path to use for the Write Ahead Log. If empty, a temporary file will be used.")
listen = flag.String("listen", ":8088", "Address to set up HTTP server listening on")
)
func main() {
- flag.Parse()
klog.InitFlags(nil)
+ flag.Parse()
+
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
@@ -75,15 +76,15 @@ type LogEntry struct {
}
func run(ctx context.Context) error {
- if *posixLogDir == "" {
- return errors.New("posix_log_dir must be set")
+ if *inputLogDir == "" {
+ return errors.New("input_log_dir must be set")
}
// Gather the info needed for reading/writing checkpoints
s, v := getSignerVerifierOrDie()
// Set up a Tessera POSIX log
- driver, err := posix.New(ctx, *posixLogDir)
+ driver, err := posix.New(ctx, posix.Config{Path: *inputLogDir})
if err != nil {
return fmt.Errorf("failed to create new log: %v", err)
}
@@ -91,6 +92,7 @@ func run(ctx context.Context) error {
// Get a Tessera appender
appender, shutdown, reader, err := tessera.NewAppender(ctx, driver, tessera.NewAppendOptions().
WithCheckpointSigner(s).
+ WithCheckpointInterval(10*time.Second).
WithBatching(256, time.Second))
if err != nil {
return fmt.Errorf("failed to get appender: %v", err)
@@ -209,21 +211,24 @@ func submitEntries(ctx context.Context, appender *tessera.Appender) {
if idx, err := appender.Add(ctx, tessera.NewEntry(data))(); err != nil {
klog.Errorf("Failed to append to log: %v", err)
} else {
- klog.Infof("Appended entry for %s@%s at index %d", module, version, idx.Index)
+ klog.V(2).Infof("Appended entry for %s@%s at index %d", module, version, idx.Index)
}
}
}
}
func runWebServer(vi *vindex.VerifiableIndex) {
- web := NewServer(func(s string) string {
- idxes := vi.Lookup(s)
- return fmt.Sprintf("Indices in log: %v", idxes)
+ web := NewServer(func(h [sha256.Size]byte) ([]uint64, error) {
+ idxes, size := vi.Lookup(h)
+ if size == 0 {
+ return nil, errors.New("index not populated")
+ }
+ return idxes, nil
})
- fs := http.FileServer(http.Dir(*posixLogDir))
+ ilfs := http.FileServer(http.Dir(*inputLogDir))
r := mux.NewRouter()
- r.PathPrefix("/log/").Handler(http.StripPrefix("/log/", fs))
+ r.PathPrefix("/inputlog/").Handler(http.StripPrefix("/inputlog/", ilfs))
web.registerHandlers(r)
hServer := &http.Server{
Addr: *listen,
diff --git a/vindex/cmd/logandmap/templates/form.html b/vindex/cmd/logandmap/templates/form.html
deleted file mode 100644
index e6c2afd..0000000
--- a/vindex/cmd/logandmap/templates/form.html
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
- Verifiable Index Lookup
-
-
-
-
-
Verifiable Index Lookup
-
{{.Message}}
-
-