From 56fcee6b885af18eb29ba1aa38fda40702c4a30c Mon Sep 17 00:00:00 2001 From: gw0 Date: Tue, 18 Nov 2025 08:39:45 +0100 Subject: [PATCH 01/22] badgerdb-sampler: Add badgerdb-sampler for BadgerDB v2 --- badgerdb-sampler/go.mod | 23 ++ badgerdb-sampler/go.sum | 113 ++++++ badgerdb-sampler/main.go | 755 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 891 insertions(+) create mode 100644 badgerdb-sampler/go.mod create mode 100644 badgerdb-sampler/go.sum create mode 100644 badgerdb-sampler/main.go diff --git a/badgerdb-sampler/go.mod b/badgerdb-sampler/go.mod new file mode 100644 index 0000000..d4fa915 --- /dev/null +++ b/badgerdb-sampler/go.mod @@ -0,0 +1,23 @@ +module consensus-blockstore-v2 + +go 1.21 + +require ( + github.com/dgraph-io/badger/v2 v2.2007.2 + github.com/gogo/protobuf v1.3.2 + github.com/tendermint/tendermint v0.34.21 +) + +require ( + github.com/DataDog/zstd v1.4.1 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect + github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + golang.org/x/net v0.0.0-20220726230323-06994584191e // indirect + golang.org/x/sys v0.0.0-20220727055044-e65921a090b8 // indirect + google.golang.org/protobuf v1.28.0 // indirect +) diff --git a/badgerdb-sampler/go.sum b/badgerdb-sampler/go.sum new file mode 100644 index 0000000..918a51a --- /dev/null +++ b/badgerdb-sampler/go.sum @@ -0,0 +1,113 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= +github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/tendermint/tendermint v0.34.21 h1:UiGGnBFHVrZhoQVQ7EfwSOLuCtarqCSsRf8VrklqB7s= +github.com/tendermint/tendermint v0.34.21/go.mod h1:XDvfg6U7grcFTDx7VkzxnhazQ/bspGJAn4DZ6DcLLjQ= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20220726230323-06994584191e h1:wOQNKh1uuDGRnmgF0jDxh7ctgGy/3P4rYWQRVJD4/Yg= +golang.org/x/net v0.0.0-20220726230323-06994584191e/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220727055044-e65921a090b8 h1:dyU22nBWzrmTQxtNrr4dzVOvaw35nUYE279vF9UmsI8= +golang.org/x/sys v0.0.0-20220727055044-e65921a090b8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/badgerdb-sampler/main.go b/badgerdb-sampler/main.go new file mode 100644 index 0000000..2b77ec6 --- /dev/null +++ b/badgerdb-sampler/main.go @@ -0,0 +1,755 @@ +package main + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + badger "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/options" + "github.com/gogo/protobuf/proto" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + tmstore "github.com/tendermint/tendermint/proto/tendermint/store" +) + +// Sample contains both raw and decoded key/value information +type Sample struct { + RawKey string `json:"raw_key"` + RawKeySize int `json:"raw_key_size"` + KeyType string `json:"key_type"` + DecodedKey string `json:"decoded_key"` + RawValue string `json:"raw_value"` + RawValueSize int `json:"raw_value_size"` + DecodedValue string `json:"decoded_value"` + Timestamp int64 `json:"timestamp,omitempty"` +} + +// DBStats contains database statistics and samples +type DBStats struct { + DatabasePath string `json:"database_path"` + DatabaseType string `json:"database_type"` + BadgerDBVersion string `json:"badgerdb_version"` + DatabaseSize int64 `json:"database_size_bytes,omitempty"` + SampleCount int `json:"sample_count"` + KeyTypeCounts map[string]int `json:"key_type_counts,omitempty"` + Samples []Sample `json:"samples"` +} + +func main() { + if len(os.Args) < 3 { + fmt.Fprintf(os.Stderr, "Usage: %s [output-subdir] [output-prefix] [max-samples]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Example: %s consensus-blockstore-v2 /path/to/blockstore.badger.db\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Example: %s runtime-mkvs-v2 /path/to/mkvs_storage.badger.db testnet-20220303 emerald 100\n", os.Args[0]) + os.Exit(1) + } + + dbType := os.Args[1] + dbPath := os.Args[2] + outputSubdir := "" + outputPrefix := "" + maxSamples := 200 // default + if len(os.Args) >= 4 { + outputSubdir = os.Args[3] + } + if len(os.Args) >= 5 { + outputPrefix = os.Args[4] + } + if len(os.Args) >= 6 { + var err error + maxSamples, err = strconv.Atoi(os.Args[5]) + if err != nil || maxSamples <= 0 { + log.Fatalf("Invalid max-samples value: %s (must be positive integer)", os.Args[5]) + } + } + + // Calculate database directory size + dbSize, err := getDatabaseSize(dbPath) + if err != nil { + log.Printf("Warning: Could not calculate database size: %v", err) + } + + // Open database with two-mode strategy (corruption handling) + db, err := openDatabase(dbPath) + if err != nil { + log.Fatalf("Failed to open database: %v", err) + } + defer db.Close() + + stats := DBStats{ + DatabasePath: dbPath, + DatabaseType: dbType, + BadgerDBVersion: "v2.2007.2", + DatabaseSize: dbSize, + KeyTypeCounts: make(map[string]int), + Samples: []Sample{}, + } + + // Collect samples + err = collectSamples(db, &stats, maxSamples) + if err != nil { + log.Fatalf("Error collecting samples: %v", err) + } + + // Output JSON + output, err := json.MarshalIndent(stats, "", " ") + if err != nil { + log.Fatalf("Error marshaling JSON: %v", err) + } + + fmt.Println(string(output)) + + // Save results in JSON files in per-snapshot dir and per-database type with optional prefix + if outputSubdir != "" { + outputDir := filepath.Join("/workspace/sampler-outputs", outputSubdir) + err = os.MkdirAll(outputDir, 0755) + if err != nil { + log.Printf("Warning: Could not create directory %s: %v", outputDir, err) + } else { + filename := outputPrefix + dbType + ".json" + outputFile := filepath.Join(outputDir, filename) + err = os.WriteFile(outputFile, output, 0644) + if err != nil { + log.Printf("Warning: Could not write to %s: %v", outputFile, err) + } else { + fmt.Fprintf(os.Stderr, "\nResults saved to: %s\n", outputFile) + } + } + } +} + +// getDatabaseSize calculates the total size of all files in the database directory +func getDatabaseSize(dbPath string) (int64, error) { + var totalSize int64 + err := filepath.Walk(dbPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + totalSize += info.Size() + } + return nil + }) + return totalSize, err +} + +// openDatabase tries to open the database in ReadOnly mode first, +// then falls back to FileIO mode for corrupted/improperly-closed databases +func openDatabase(path string) (*badger.DB, error) { + // Try ReadOnly mode first (clean databases) + opts := badger.DefaultOptions(path) + opts.ReadOnly = true + opts.Logger = nil + + db, err := badger.Open(opts) + if err == nil { + fmt.Fprintf(os.Stderr, "Opened database in ReadOnly mode\n") + return db, nil + } + + // Check for BadgerDB v3 incompatibility + if strings.Contains(err.Error(), "manifest has unsupported version") { + return nil, fmt.Errorf("database uses BadgerDB v3 (incompatible). Please use sampler-v3 instead.\nOriginal error: %v", err) + } + + fmt.Fprintf(os.Stderr, "ReadOnly mode failed, trying FileIO workaround...\n") + + // FileIO workaround mode - for improperly closed databases + opts = badger.DefaultOptions(path) + opts.ReadOnly = false // Must use read-write for improperly closed DB + opts.SyncWrites = false // Don't sync writes + opts.NumMemtables = 5 // More memtables to avoid flushing + opts.NumLevelZeroTables = 100 // High threshold to avoid compaction + opts.NumLevelZeroTablesStall = 200 // Very high stall threshold + opts.NumCompactors = 0 // Disable background compaction + opts.CompactL0OnClose = false // Don't compact on close + opts.ValueLogLoadingMode = options.FileIO // Use FileIO instead of mmap + opts.TableLoadingMode = options.FileIO // Use FileIO instead of mmap + opts.Logger = nil + + db, err = badger.Open(opts) + if err != nil { + // Check again for v3 incompatibility in FileIO mode + if strings.Contains(err.Error(), "manifest has unsupported version") { + return nil, fmt.Errorf("database uses BadgerDB v3 (incompatible). Please use sampler-v3 instead.\nOriginal error: %v", err) + } + return nil, fmt.Errorf("failed to open database with both ReadOnly and FileIO modes: %v", err) + } + + fmt.Fprintf(os.Stderr, "Opened database in FileIO mode\n") + return db, nil +} + +// collectSamples iterates through keys and collects samples with full decoding +func collectSamples(db *badger.DB, stats *DBStats, maxSamples int) error { + return db.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.PrefetchValues = true // Fetch values for sampling + opts.PrefetchSize = maxSamples // Only prefetch what we need + it := txn.NewIterator(opts) + defer it.Close() + + sampleCount := 0 + + for it.Rewind(); it.Valid(); it.Next() { + // Only collect samples up to maxSamples + if sampleCount >= maxSamples { + break + } + item := it.Item() + key := item.Key() + + sample := Sample{ + RawKey: fmt.Sprintf("%x", key), + RawKeySize: len(key), + KeyType: "unknown", + DecodedKey: "", + } + + // Analyze key based on database type + switch stats.DatabaseType { + case "consensus-blockstore-v2": + keyType, decodedKey := analyzeKeyConsensusBlockstoreV2(key) + sample.KeyType = keyType + sample.DecodedKey = decodedKey + case "consensus-evidence-v2": + keyType, decodedKey := analyzeKeyConsensusEvidenceV2(key) + sample.KeyType = keyType + sample.DecodedKey = decodedKey + case "consensus-mkvs-v2": + keyType, decodedKey := analyzeKeyConsensusMkvsV2(key) + sample.KeyType = keyType + sample.DecodedKey = decodedKey + case "consensus-state-v2": + keyType, decodedKey := analyzeKeyConsensusStateV2(key) + sample.KeyType = keyType + sample.DecodedKey = decodedKey + case "runtime-mkvs-v2": + keyType, decodedKey := analyzeKeyRuntimeMkvsV2(key) + sample.KeyType = keyType + sample.DecodedKey = decodedKey + case "runtime-history-v2": + keyType, decodedKey := analyzeKeyRuntimeHistoryV2(key) + sample.KeyType = keyType + sample.DecodedKey = decodedKey + } + + // Fetch and decode value + err := item.Value(func(val []byte) error { + sample.RawValueSize = len(val) + sample.RawValue = truncateHex(val, 100) + sample.DecodedValue = "" + + // Decode value based on database type + switch stats.DatabaseType { + case "consensus-blockstore-v2": + decoded, ts := decodeValueConsensusBlockstoreV2(sample.KeyType, val) + sample.DecodedValue = decoded + sample.Timestamp = ts + case "consensus-evidence-v2": + sample.DecodedValue = decodeValueConsensusEvidenceV2(sample.KeyType, val) + case "consensus-mkvs-v2": + sample.DecodedValue = decodeValueConsensusMkvsV2(sample.KeyType, val) + case "consensus-state-v2": + sample.DecodedValue = decodeValueConsensusStateV2(sample.KeyType, val) + case "runtime-mkvs-v2": + sample.DecodedValue = decodeValueRuntimeMkvsV2(sample.KeyType, val) + case "runtime-history-v2": + sample.DecodedValue = decodeValueRuntimeHistoryV2(sample.KeyType, val) + } + return nil + }) + if err != nil { + log.Printf("Error reading value for key %s: %v", sample.RawKey, err) + } + + stats.Samples = append(stats.Samples, sample) + stats.KeyTypeCounts[sample.KeyType]++ + sampleCount++ + } + + stats.SampleCount = sampleCount + return nil + }) +} + +// analyzeKeyConsensusBlockstoreV2 parses consensus-blockstore-v2 key and returns key type and decoded representation +func analyzeKeyConsensusBlockstoreV2(key []byte) (string, string) { + if len(key) < 2 { + return "unknown", fmt.Sprintf("%x", key) + } + + // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) + if key[0] != 0x01 { + return "unknown", fmt.Sprintf("%x", key) + } + + // Parse ASCII key after 0x01 prefix + asciiKey := string(key[1:]) + + // blockStore state key + if asciiKey == "blockStore" { + return "blockstore_state", "blockStore" + } + + // Parse structured keys: "H:{height}", "P:{height}:{part}", "C:{height}", "SC:{height}", "BH:{hash}" + if strings.HasPrefix(asciiKey, "H:") { + return "block_meta", asciiKey + } + if strings.HasPrefix(asciiKey, "P:") { + return "block_part", asciiKey + } + if strings.HasPrefix(asciiKey, "C:") { + return "block_commit", asciiKey + } + if strings.HasPrefix(asciiKey, "SC:") { + return "seen_commit", asciiKey + } + if strings.HasPrefix(asciiKey, "BH:") { + return "block_hash", asciiKey + } + + return "unknown", asciiKey +} + +// decodeValueConsensusBlockstoreV2 attempts to decode protobuf value based on key type +// Returns: (decoded_string, timestamp_unix) +func decodeValueConsensusBlockstoreV2(keyType string, value []byte) (string, int64) { + switch keyType { + case "blockstore_state": + var state tmstore.BlockStoreState + if err := proto.Unmarshal(value, &state); err == nil { + return fmt.Sprintf("BlockStoreState{base: %d, height: %d}", state.Base, state.Height), 0 + } + return fmt.Sprintf("Failed to decode BlockStoreState (size: %d bytes)", len(value)), 0 + + case "block_meta": + var meta tmproto.BlockMeta + if err := proto.Unmarshal(value, &meta); err == nil { + // Extract timestamp + tsUnix := int64(0) + ts := "" + if meta.Header.Time.Unix() > 0 { + tsUnix = meta.Header.Time.Unix() + ts = meta.Header.Time.Format(time.RFC3339) + } + + // Format AppHash and ChainID + appHash := "" + if len(meta.Header.AppHash) >= 8 { + appHash = fmt.Sprintf("%x", meta.Header.AppHash[:8]) + } else if len(meta.Header.AppHash) > 0 { + appHash = fmt.Sprintf("%x", meta.Header.AppHash) + } + + chainID := meta.Header.ChainID + if len(chainID) > 20 { + chainID = chainID[:20] + "..." + } + + return fmt.Sprintf("BlockMeta{height: %d, time: %s, chain: %s, num_txs: %d, app_hash: %s}", + meta.Header.Height, ts, chainID, meta.NumTxs, appHash), tsUnix + } + return fmt.Sprintf("Failed to decode BlockMeta (size: %d bytes)", len(value)), 0 + + case "block_part": + var part tmproto.Part + if err := proto.Unmarshal(value, &part); err == nil { + return fmt.Sprintf("Part{index: %d, bytes_size: %d, proof_total: %d}", + part.Index, len(part.Bytes), part.Proof.Total), 0 + } + return fmt.Sprintf("Failed to decode Part (size: %d bytes)", len(value)), 0 + + case "block_commit": + var commit tmproto.Commit + if err := proto.Unmarshal(value, &commit); err == nil { + // Commits don't have a direct timestamp field; timestamp comes from block header + return fmt.Sprintf("Commit{height: %d, round: %d, signatures: %d}", + commit.Height, commit.Round, len(commit.Signatures)), 0 + } + return fmt.Sprintf("Failed to decode Commit (size: %d bytes)", len(value)), 0 + + case "seen_commit": + var commit tmproto.Commit + if err := proto.Unmarshal(value, &commit); err == nil { + // Commits don't have a direct timestamp field; timestamp comes from block header + return fmt.Sprintf("SeenCommit{height: %d, round: %d, signatures: %d}", + commit.Height, commit.Round, len(commit.Signatures)), 0 + } + return fmt.Sprintf("Failed to decode SeenCommit (size: %d bytes)", len(value)), 0 + + case "block_hash": + // Block hash values are plain strings containing height numbers + height, err := strconv.ParseInt(string(value), 10, 64) + if err == nil { + return fmt.Sprintf("Height: %d", height), 0 + } + return fmt.Sprintf("String: %s", string(value)), 0 + + default: + return fmt.Sprintf("Unknown type (size: %d bytes)", len(value)), 0 + } +} + +// analyzeKeyConsensusEvidenceV2 parses consensus-evidence-v2 key and returns key type and decoded representation +func analyzeKeyConsensusEvidenceV2(key []byte) (string, string) { + if len(key) < 2 { + return "unknown", fmt.Sprintf("%x", key) + } + + // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) + if key[0] != 0x01 { + return "unknown", fmt.Sprintf("%x", key) + } + + // Evidence DB is typically empty or uses simple key patterns + // Return the hex representation for now + return fmt.Sprintf("type_%02x", key[1]), fmt.Sprintf("%x", key[1:]) +} + +// decodeValueConsensusEvidenceV2 attempts to decode evidence value +func decodeValueConsensusEvidenceV2(keyType string, value []byte) string { + // Evidence values are protobuf-encoded DuplicateVoteEvidence or LightClientAttackEvidence + // For now, just show size as evidence DB is typically empty + if len(value) == 0 { + return "Empty" + } + + // Try to decode as DuplicateVoteEvidence + var evidence tmproto.DuplicateVoteEvidence + if err := proto.Unmarshal(value, &evidence); err == nil { + return fmt.Sprintf("DuplicateVoteEvidence{vote_a_height: %d, vote_b_height: %d}", + evidence.VoteA.Height, evidence.VoteB.Height) + } + + return fmt.Sprintf("Evidence (size: %d bytes, type: %s)", len(value), keyType) +} + +// analyzeKeyConsensusMkvsV2 parses consensus-mkvs-v2 key and returns key type and decoded representation +// MKVS keys: 0xNN (type prefix) + type-specific data (NO dbVersion prefix) +// Key formats from oasis-core/go/storage/mkvs/db/badger/badger.go: +// 0x00: node (hash) +// 0x01: writeLog (version uint64, new root, old root) +// 0x02: rootsMetadata (version uint64) +// 0x03: rootUpdatedNodes (version uint64, root) +// 0x04: metadata +// 0x05: multipartRestoreNodeLog (hash) +// 0x06: rootNode (typed hash) +func analyzeKeyConsensusMkvsV2(key []byte) (string, string) { + if len(key) < 1 { + return "unknown", fmt.Sprintf("%x", key) + } + + // MKVS keys start directly with the type prefix byte (0x00-0x06) + // NO dbVersion prefix for MKVS storage + prefixByte := key[0] + data := key[1:] + + switch prefixByte { + case 0x00: + // nodeKeyFmt: hash.Hash (32 bytes) + if len(data) >= 32 { + return "node", fmt.Sprintf("node:hash=%x", data[:32]) + } + return "node", fmt.Sprintf("node:hash=%x", data) + case 0x01: + // writeLogKeyFmt: uint64(version) + TypedHash(new root) + TypedHash(old root) + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + if len(data) >= 41 { + return "write_log", fmt.Sprintf("write_log:v=%d,new_root=%x,old_root=%x", + version, truncateBytes(data[8:41], 8), truncateBytes(data[41:], 8)) + } + return "write_log", fmt.Sprintf("write_log:v=%d", version) + } + return "write_log", "write_log:" + case 0x02: + // rootsMetadataKeyFmt: uint64(version) + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + return "roots_metadata", fmt.Sprintf("roots_metadata:v=%d", version) + } + return "roots_metadata", "roots_metadata:" + case 0x03: + // rootUpdatedNodesKeyFmt: uint64(version) + TypedHash(root) + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + if len(data) >= 41 { + return "root_updated_nodes", fmt.Sprintf("root_updated_nodes:v=%d,root=%x", + version, truncateBytes(data[8:41], 8)) + } + return "root_updated_nodes", fmt.Sprintf("root_updated_nodes:v=%d", version) + } + return "root_updated_nodes", "root_updated_nodes:" + case 0x04: + // metadataKeyFmt: no additional data + return "metadata", "metadata" + case 0x05: + // multipartRestoreNodeLogKeyFmt: TypedHash (33 bytes: type byte + 32-byte hash) + if len(data) >= 33 { + return "multipart_restore_log", fmt.Sprintf("multipart_restore_log:hash=%x", data[:33]) + } + return "multipart_restore_log", fmt.Sprintf("multipart_restore_log:hash=%x", data) + case 0x06: + // rootNodeKeyFmt: TypedHash (33 bytes: type byte + 32-byte hash) + if len(data) >= 33 { + return "root_node", fmt.Sprintf("root_node:hash=%x", data[:33]) + } + return "root_node", fmt.Sprintf("root_node:hash=%x", data) + default: + return fmt.Sprintf("unknown_%02x", prefixByte), fmt.Sprintf("unknown_%02x:%x", prefixByte, truncateBytes(data, 8)) + } +} + +// decodeValueConsensusMkvsV2 decodes MKVS value +// Value types: +// - node: Serialized node data (binary) +// - write_log: CBOR-serialized write log +// - roots_metadata: CBOR-serialized rootsMetadata +// - root_updated_nodes: CBOR-serialized []updatedNode +// - metadata: CBOR-serialized metadata +func decodeValueConsensusMkvsV2(keyType string, value []byte) string { + if len(value) == 0 { + return "Empty" + } + + // For CBOR-encoded values, we could decode but it's complex + // Just report size and encoding type + switch keyType { + case "node": + return fmt.Sprintf("Serialized node (size: %d bytes)", len(value)) + case "write_log", "roots_metadata", "root_updated_nodes", "metadata": + return fmt.Sprintf("CBOR-encoded %s (size: %d bytes)", keyType, len(value)) + default: + return fmt.Sprintf("MKVS %s data (size: %d bytes)", keyType, len(value)) + } +} + +// analyzeKeyConsensusStateV2 parses consensus-state-v2 key and returns key type and decoded representation +// State keys are ASCII strings: 0x01 + ":" +func analyzeKeyConsensusStateV2(key []byte) (string, string) { + if len(key) < 2 { + return "unknown", fmt.Sprintf("%x", key) + } + + // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) + if key[0] != 0x01 { + return "unknown", fmt.Sprintf("%x", key) + } + + // Parse ASCII key after 0x01 prefix + decoded := string(key[1:]) + + // Check if it's printable ASCII + if !isPrintableASCII(decoded) { + return "binary", fmt.Sprintf("%x", key) + } + + // Extract key type prefix before colon + colonIdx := strings.IndexByte(decoded, ':') + if colonIdx != -1 { + prefix := decoded[:colonIdx] + switch prefix { + case "abciResponsesKey": + return "abci_responses", decoded + case "consensusParamsKey": + return "consensus_params", decoded + case "validatorsKey": + return "validators", decoded + case "stateKey": + return "state", decoded + case "genesisDoc": + return "genesis", decoded + default: + return prefix, decoded + } + } + + return "text_key", decoded +} + +// decodeValueConsensusStateV2 decodes state value (protobuf-encoded Tendermint state data) +func decodeValueConsensusStateV2(keyType string, value []byte) string { + // State values are protobuf-encoded, but the exact message type depends on the key + // For simplicity, just show size and type info + if len(value) == 0 { + return "Empty" + } + + return fmt.Sprintf("Tendermint %s data (size: %d bytes)", keyType, len(value)) +} + +// analyzeKeyRuntimeMkvsV2 parses runtime-mkvs-v2 key and returns key type and decoded representation +// MKVS keys: 0xNN (type prefix) + type-specific data (NO dbVersion prefix) +// Same format as consensus-mkvs-v2 from oasis-core/go/storage/mkvs/db/badger/badger.go +func analyzeKeyRuntimeMkvsV2(key []byte) (string, string) { + if len(key) < 1 { + return "unknown", fmt.Sprintf("%x", key) + } + + // MKVS keys start directly with the type prefix byte (0x00-0x06) + // NO dbVersion prefix for MKVS storage + prefixByte := key[0] + data := key[1:] + + switch prefixByte { + case 0x00: + // nodeKeyFmt: hash.Hash (32 bytes) + if len(data) >= 32 { + return "node", fmt.Sprintf("node:hash=%x", data[:32]) + } + return "node", fmt.Sprintf("node:hash=%x", data) + case 0x01: + // writeLogKeyFmt: uint64(version) + TypedHash(new root) + TypedHash(old root) + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + if len(data) >= 41 { + return "write_log", fmt.Sprintf("write_log:v=%d,new_root=%x,old_root=%x", + version, truncateBytes(data[8:41], 8), truncateBytes(data[41:], 8)) + } + return "write_log", fmt.Sprintf("write_log:v=%d", version) + } + return "write_log", "write_log:" + case 0x02: + // rootsMetadataKeyFmt: uint64(version) + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + return "roots_metadata", fmt.Sprintf("roots_metadata:v=%d", version) + } + return "roots_metadata", "roots_metadata:" + case 0x03: + // rootUpdatedNodesKeyFmt: uint64(version) + TypedHash(root) + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + if len(data) >= 41 { + return "root_updated_nodes", fmt.Sprintf("root_updated_nodes:v=%d,root=%x", + version, truncateBytes(data[8:41], 8)) + } + return "root_updated_nodes", fmt.Sprintf("root_updated_nodes:v=%d", version) + } + return "root_updated_nodes", "root_updated_nodes:" + case 0x04: + // metadataKeyFmt: no additional data + return "metadata", "metadata" + case 0x05: + // multipartRestoreNodeLogKeyFmt: TypedHash (33 bytes: type byte + 32-byte hash) + if len(data) >= 33 { + return "multipart_restore_log", fmt.Sprintf("multipart_restore_log:hash=%x", data[:33]) + } + return "multipart_restore_log", fmt.Sprintf("multipart_restore_log:hash=%x", data) + case 0x06: + // rootNodeKeyFmt: TypedHash (33 bytes: type byte + 32-byte hash) + if len(data) >= 33 { + return "root_node", fmt.Sprintf("root_node:hash=%x", data[:33]) + } + return "root_node", fmt.Sprintf("root_node:hash=%x", data) + default: + return fmt.Sprintf("unknown_%02x", prefixByte), fmt.Sprintf("unknown_%02x:%x", prefixByte, truncateBytes(data, 8)) + } +} + +// decodeValueRuntimeMkvsV2 decodes runtime MKVS value +// Same format as consensus MKVS (CBOR-encoded for most types) +func decodeValueRuntimeMkvsV2(keyType string, value []byte) string { + if len(value) == 0 { + return "Empty" + } + + switch keyType { + case "node": + return fmt.Sprintf("Serialized node (size: %d bytes)", len(value)) + case "write_log", "roots_metadata", "root_updated_nodes", "metadata": + return fmt.Sprintf("CBOR-encoded %s (size: %d bytes)", keyType, len(value)) + default: + return fmt.Sprintf("MKVS %s data (size: %d bytes)", keyType, len(value)) + } +} + +// analyzeKeyRuntimeHistoryV2 parses runtime-history-v2 key and returns key type and decoded representation +// History keys are ASCII strings: 0x01 + ":" +func analyzeKeyRuntimeHistoryV2(key []byte) (string, string) { + if len(key) < 2 { + return "unknown", fmt.Sprintf("%x", key) + } + + // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) + if key[0] != 0x01 { + return "unknown", fmt.Sprintf("%x", key) + } + + // Parse ASCII key after 0x01 prefix + decoded := string(key[1:]) + + // Check if it's printable ASCII + if !isPrintableASCII(decoded) { + return "binary", fmt.Sprintf("%x", key) + } + + // Extract key type prefix before colon + colonIdx := strings.IndexByte(decoded, ':') + if colonIdx != -1 { + prefix := decoded[:colonIdx] + switch prefix { + case "abciResponsesKey": + return "abci_responses", decoded + case "consensusParamsKey": + return "consensus_params", decoded + case "validatorsKey": + return "validators", decoded + case "stateKey": + return "state", decoded + case "genesisDoc": + return "genesis", decoded + default: + return prefix, decoded + } + } + + return "text_key", decoded +} + +// decodeValueRuntimeHistoryV2 decodes runtime history value +func decodeValueRuntimeHistoryV2(keyType string, value []byte) string { + // Runtime history values are similar to consensus state + // For simplicity, just show size and type info + if len(value) == 0 { + return "Empty" + } + + return fmt.Sprintf("Runtime %s data (size: %d bytes)", keyType, len(value)) +} + +// isPrintableASCII checks if a string contains only printable ASCII characters +func isPrintableASCII(s string) bool { + if len(s) == 0 { + return false + } + for _, r := range s { + if r < 32 || r > 126 { + return false + } + } + return true +} + +// truncateBytes returns first n bytes or all bytes if shorter +func truncateBytes(data []byte, n int) []byte { + if len(data) <= n { + return data + } + return data[:n] +} + +// truncateHex converts bytes to hex and truncates if too long +func truncateHex(data []byte, maxLen int) string { + hex := fmt.Sprintf("%x", data) + if len(hex) > maxLen { + return hex[:maxLen] + "..." + } + return hex +} From cb31e1c5aaffe8d1fe739bef814c799fcc07337b Mon Sep 17 00:00:00 2001 From: gw0 Date: Tue, 18 Nov 2025 12:31:42 +0100 Subject: [PATCH 02/22] badgerdb-sampler: Refactor and add support for BadgerDB v3 --- badgerdb-sampler/Makefile | 38 ++++ badgerdb-sampler/README.md | 44 ++++ badgerdb-sampler/db_v2.go | 73 +++++++ badgerdb-sampler/db_v3.go | 73 +++++++ badgerdb-sampler/db_v4.go | 72 +++++++ badgerdb-sampler/go.mod | 21 +- badgerdb-sampler/go.sum | 112 +++++++++-- badgerdb-sampler/go_v2.mod | 23 +++ badgerdb-sampler/go_v2.sum | 113 +++++++++++ badgerdb-sampler/go_v3.mod | 27 +++ badgerdb-sampler/go_v3.sum | 197 ++++++++++++++++++ badgerdb-sampler/go_v4.mod | 12 ++ badgerdb-sampler/main.go | 402 +++++++++++-------------------------- 13 files changed, 894 insertions(+), 313 deletions(-) create mode 100644 badgerdb-sampler/Makefile create mode 100644 badgerdb-sampler/README.md create mode 100644 badgerdb-sampler/db_v2.go create mode 100644 badgerdb-sampler/db_v3.go create mode 100644 badgerdb-sampler/db_v4.go create mode 100644 badgerdb-sampler/go_v2.mod create mode 100644 badgerdb-sampler/go_v2.sum create mode 100644 badgerdb-sampler/go_v3.mod create mode 100644 badgerdb-sampler/go_v3.sum create mode 100644 badgerdb-sampler/go_v4.mod diff --git a/badgerdb-sampler/Makefile b/badgerdb-sampler/Makefile new file mode 100644 index 0000000..a9642ca --- /dev/null +++ b/badgerdb-sampler/Makefile @@ -0,0 +1,38 @@ +# Makefile for badgerdb-sampler + +.PHONY: all build-all build-v2 build-v3 build-v4 clean + +# Default target +all: build-all + +# Build all versions +build-all: build-v2 build-v3 build-v4 + +# Build BadgerDB v2 version +build-v2: + @echo "Building badgerdb-sampler-v2..." + @cp go_v2.mod go.mod && cp go_v2.sum go.sum 2>/dev/null || true + go build -tags badgerv2 -o bin/badgerdb-sampler-v2 + @echo "✓ Built: bin/badgerdb-sampler-v2" + +# Build BadgerDB v3 version +build-v3: + @echo "Building badgerdb-sampler-v3..." + @cp go_v3.mod go.mod && cp go_v3.sum go.sum 2>/dev/null || true + go build -tags badgerv3 -o bin/badgerdb-sampler-v3 + @echo "✓ Built: bin/badgerdb-sampler-v3" + +# Build BadgerDB v4 version +build-v4: + @echo "Building badgerdb-sampler-v4..." + @cp go_v4.mod go.mod && cp go_v4.sum go.sum 2>/dev/null || true + go build -tags badgerv4 -o bin/badgerdb-sampler-v4 + @echo "✓ Built: bin/badgerdb-sampler-v4" + +# Clean build artifacts +clean: + @echo "Cleaning bin/..." + rm -f bin/badgerdb-sampler-v2 bin/badgerdb-sampler-v3 bin/badgerdb-sampler-v4 + @echo "Cleaning outputs/..." + rm -rf outputs/* + @echo "✓ Cleaned" diff --git a/badgerdb-sampler/README.md b/badgerdb-sampler/README.md new file mode 100644 index 0000000..4b8c163 --- /dev/null +++ b/badgerdb-sampler/README.md @@ -0,0 +1,44 @@ +# BadgerDB Sampler + +A tool for sampling BadgerDB v2, v3, and v4 databases used by Oasis nodes. + +## Building + +```bash +# Build all versions +make build-all +``` + +## Usage + +### Prerequisites + +- Go 1.21 or higher +- BadgerDB databases from Oasis nodes (see https://snapshots.oasis.io/) + +### Run + +```bash +# Same interface for all versions +./badgerdb-sampler-v2 [output-subdir] [output-prefix] [max-samples] +./badgerdb-sampler-v3 [output-subdir] [output-prefix] [max-samples] +``` + +Database types (version suffixes are optional): + +- `consensus-blockstore` (or `consensus-blockstore-v2`, `consensus-blockstore-v3`) +- `consensus-evidence` +- `consensus-mkvs` +- `consensus-state` +- `runtime-mkvs` +- `runtime-history` + +### Examples + +```bash +# Analyze v2 consensus blockstore +./badgerdb-sampler-v2 consensus-blockstore /path/to/blockstore.badger.db ./outputs/testnet-20220303/ + +# Analyze v3 runtime MKVS +./badgerdb-sampler-v3 runtime-mkvs /path/to/mkvs_storage.badger.db ./outputs/testnet-20220303/emerald- 500 +``` diff --git a/badgerdb-sampler/db_v2.go b/badgerdb-sampler/db_v2.go new file mode 100644 index 0000000..1c614c4 --- /dev/null +++ b/badgerdb-sampler/db_v2.go @@ -0,0 +1,73 @@ +//go:build badgerv2 + +package main + +import ( + "fmt" + "os" + "strings" + + badger "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/options" +) + +type ( + DB = badger.DB + Txn = badger.Txn + Item = badger.Item + Iterator = badger.Iterator + IteratorOptions = badger.IteratorOptions +) + +var DefaultIteratorOptions = badger.DefaultIteratorOptions + +func init() { + BadgerVersion = "v2.2007.2" +} + +// openDatabase tries to open the database in ReadOnly mode first, +// then falls back to FileIO mode for corrupted/improperly-closed databases +func openDatabase(path string) (*DB, error) { + // Try ReadOnly mode first (clean databases) + opts := badger.DefaultOptions(path) + opts.ReadOnly = true + opts.Logger = nil + + db, err := badger.Open(opts) + if err == nil { + fmt.Fprintf(os.Stderr, "Opened database in ReadOnly mode\n") + return db, nil + } + + // Check for BadgerDB v3/v4 incompatibility + if strings.Contains(err.Error(), "manifest has unsupported version") { + return nil, fmt.Errorf("database uses BadgerDB v3 or v4 (incompatible). Please use badgerdb-sampler-v3 or badgerdb-sampler-v4 instead.\nOriginal error: %v", err) + } + + fmt.Fprintf(os.Stderr, "ReadOnly mode failed, trying FileIO workaround...\n") + + // FileIO workaround mode - for improperly closed databases + opts = badger.DefaultOptions(path) + opts.ReadOnly = false + opts.SyncWrites = false + opts.NumMemtables = 5 + opts.NumLevelZeroTables = 100 + opts.NumLevelZeroTablesStall = 200 + opts.NumCompactors = 0 + opts.CompactL0OnClose = false + opts.ValueLogLoadingMode = options.FileIO + opts.TableLoadingMode = options.FileIO + opts.Logger = nil + + db, err = badger.Open(opts) + if err != nil { + // Check again for v3/v4 incompatibility in FileIO mode + if strings.Contains(err.Error(), "manifest has unsupported version") { + return nil, fmt.Errorf("database uses BadgerDB v3 or v4 (incompatible). Please use badgerdb-sampler-v3 or badgerdb-sampler-v4 instead.\nOriginal error: %v", err) + } + return nil, fmt.Errorf("failed to open database with both ReadOnly and FileIO modes: %v", err) + } + + fmt.Fprintf(os.Stderr, "Opened database in FileIO mode\n") + return db, nil +} diff --git a/badgerdb-sampler/db_v3.go b/badgerdb-sampler/db_v3.go new file mode 100644 index 0000000..8b0a2c0 --- /dev/null +++ b/badgerdb-sampler/db_v3.go @@ -0,0 +1,73 @@ +//go:build badgerv3 + +package main + +import ( + "fmt" + "os" + "strings" + + badger "github.com/dgraph-io/badger/v3" +) + +type ( + DB = badger.DB + Txn = badger.Txn + Item = badger.Item + Iterator = badger.Iterator + IteratorOptions = badger.IteratorOptions +) + +var DefaultIteratorOptions = badger.DefaultIteratorOptions + +func init() { + BadgerVersion = "v3.2103.5" +} + +// openDatabase tries to open the database in ReadOnly mode first, +// then falls back to FileIO mode for corrupted/improperly-closed databases +func openDatabase(path string) (*DB, error) { + // Try ReadOnly mode first (clean databases) + opts := badger.DefaultOptions(path) + opts.ReadOnly = true + opts.Logger = nil + + db, err := badger.Open(opts) + if err == nil { + fmt.Fprintf(os.Stderr, "Opened database in ReadOnly mode\n") + return db, nil + } + + // Check for BadgerDB v2/v4 incompatibility + if strings.Contains(err.Error(), "manifest has unsupported version") || + strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database may use BadgerDB v2 or v4 (incompatible). Please use badgerdb-sampler-v2 or badgerdb-sampler-v4 instead.\nOriginal error: %v", err) + } + + fmt.Fprintf(os.Stderr, "ReadOnly mode failed, trying FileIO workaround...\n") + + // FileIO workaround mode - for improperly closed databases + opts = badger.DefaultOptions(path) + opts.ReadOnly = false + opts.SyncWrites = false + opts.NumMemtables = 5 + opts.NumLevelZeroTables = 100 + opts.NumLevelZeroTablesStall = 200 + opts.NumCompactors = 0 + opts.CompactL0OnClose = false + // Note: ValueLogLoadingMode and TableLoadingMode were removed in BadgerDB v3 + opts.Logger = nil + + db, err = badger.Open(opts) + if err != nil { + // Check again for v2/v4 incompatibility in FileIO mode + if strings.Contains(err.Error(), "manifest has unsupported version") || + strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database may use BadgerDB v2 or v4 (incompatible). Please use badgerdb-sampler-v2 or badgerdb-sampler-v4 instead.\nOriginal error: %v", err) + } + return nil, fmt.Errorf("failed to open database with both ReadOnly and FileIO modes: %v", err) + } + + fmt.Fprintf(os.Stderr, "Opened database in FileIO mode\n") + return db, nil +} diff --git a/badgerdb-sampler/db_v4.go b/badgerdb-sampler/db_v4.go new file mode 100644 index 0000000..27a35e5 --- /dev/null +++ b/badgerdb-sampler/db_v4.go @@ -0,0 +1,72 @@ +//go:build badgerv4 + +package main + +import ( + "fmt" + "os" + "strings" + + badger "github.com/dgraph-io/badger/v4" +) + +type ( + DB = badger.DB + Txn = badger.Txn + Item = badger.Item + Iterator = badger.Iterator + IteratorOptions = badger.IteratorOptions +) + +var DefaultIteratorOptions = badger.DefaultIteratorOptions + +func init() { + BadgerVersion = "v4.x.x" +} + +// openDatabase tries to open the database in ReadOnly mode first, +// then falls back to FileIO mode for corrupted/improperly-closed databases +func openDatabase(path string) (*DB, error) { + // Try ReadOnly mode first (clean databases) + opts := badger.DefaultOptions(path) + opts.ReadOnly = true + opts.Logger = nil + + db, err := badger.Open(opts) + if err == nil { + fmt.Fprintf(os.Stderr, "Opened database in ReadOnly mode\n") + return db, nil + } + + // Check for BadgerDB v2/v3 incompatibility + if strings.Contains(err.Error(), "manifest has unsupported version") || + strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database may use BadgerDB v2 or v3 (incompatible). Please use badgerdb-sampler-v2 or badgerdb-sampler-v3 instead.\nOriginal error: %v", err) + } + + fmt.Fprintf(os.Stderr, "ReadOnly mode failed, trying FileIO workaround...\n") + + // FileIO workaround mode - for improperly closed databases + opts = badger.DefaultOptions(path) + opts.ReadOnly = false + opts.SyncWrites = false + opts.NumMemtables = 5 + opts.NumLevelZeroTables = 100 + opts.NumLevelZeroTablesStall = 200 + opts.NumCompactors = 0 + opts.CompactL0OnClose = false + opts.Logger = nil + + db, err = badger.Open(opts) + if err != nil { + // Check again for v2/v3 incompatibility in FileIO mode + if strings.Contains(err.Error(), "manifest has unsupported version") || + strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database may use BadgerDB v2 or v3 (incompatible). Please use badgerdb-sampler-v2 or badgerdb-sampler-v3 instead.\nOriginal error: %v", err) + } + return nil, fmt.Errorf("failed to open database with both ReadOnly and FileIO modes: %v", err) + } + + fmt.Fprintf(os.Stderr, "Opened database in FileIO mode\n") + return db, nil +} diff --git a/badgerdb-sampler/go.mod b/badgerdb-sampler/go.mod index d4fa915..f9332c6 100644 --- a/badgerdb-sampler/go.mod +++ b/badgerdb-sampler/go.mod @@ -1,23 +1,12 @@ -module consensus-blockstore-v2 +module github.com/oasisprotocol/badgerdb-sampler -go 1.21 +go 1.25.4 require ( - github.com/dgraph-io/badger/v2 v2.2007.2 + github.com/dgraph-io/badger/v4 v4.2.0 github.com/gogo/protobuf v1.3.2 github.com/tendermint/tendermint v0.34.21 ) -require ( - github.com/DataDog/zstd v1.4.1 // indirect - github.com/cespare/xxhash v1.1.0 // indirect - github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect - github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/golang/snappy v0.0.1 // indirect - github.com/pkg/errors v0.9.1 // indirect - golang.org/x/net v0.0.0-20220726230323-06994584191e // indirect - golang.org/x/sys v0.0.0-20220727055044-e65921a090b8 // indirect - google.golang.org/protobuf v1.28.0 // indirect -) +// Note: Dependencies will be filled in when v4 support is implemented +// Run 'go mod tidy' with -modfile=go.mod.v4 to populate diff --git a/badgerdb-sampler/go.sum b/badgerdb-sampler/go.sum index 918a51a..4ffd7ff 100644 --- a/badgerdb-sampler/go.sum +++ b/badgerdb-sampler/go.sum @@ -1,11 +1,16 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= -github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -13,30 +18,63 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= -github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= +github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -44,11 +82,11 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -62,6 +100,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/tendermint/tendermint v0.34.21 h1:UiGGnBFHVrZhoQVQ7EfwSOLuCtarqCSsRf8VrklqB7s= @@ -70,31 +109,52 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20220726230323-06994584191e h1:wOQNKh1uuDGRnmgF0jDxh7ctgGy/3P4rYWQRVJD4/Yg= golang.org/x/net v0.0.0-20220726230323-06994584191e/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220727055044-e65921a090b8 h1:dyU22nBWzrmTQxtNrr4dzVOvaw35nUYE279vF9UmsI8= -golang.org/x/sys v0.0.0-20220727055044-e65921a090b8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -102,6 +162,27 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= @@ -109,5 +190,8 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/badgerdb-sampler/go_v2.mod b/badgerdb-sampler/go_v2.mod new file mode 100644 index 0000000..c493ddd --- /dev/null +++ b/badgerdb-sampler/go_v2.mod @@ -0,0 +1,23 @@ +module github.com/oasisprotocol/badgerdb-sampler + +go 1.21 + +require ( + github.com/dgraph-io/badger/v2 v2.2007.2 + github.com/gogo/protobuf v1.3.2 + github.com/tendermint/tendermint v0.34.21 +) + +require ( + github.com/DataDog/zstd v1.4.1 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect + github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + golang.org/x/net v0.0.0-20220726230323-06994584191e // indirect + golang.org/x/sys v0.0.0-20220727055044-e65921a090b8 // indirect + google.golang.org/protobuf v1.28.0 // indirect +) diff --git a/badgerdb-sampler/go_v2.sum b/badgerdb-sampler/go_v2.sum new file mode 100644 index 0000000..918a51a --- /dev/null +++ b/badgerdb-sampler/go_v2.sum @@ -0,0 +1,113 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= +github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/tendermint/tendermint v0.34.21 h1:UiGGnBFHVrZhoQVQ7EfwSOLuCtarqCSsRf8VrklqB7s= +github.com/tendermint/tendermint v0.34.21/go.mod h1:XDvfg6U7grcFTDx7VkzxnhazQ/bspGJAn4DZ6DcLLjQ= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20220726230323-06994584191e h1:wOQNKh1uuDGRnmgF0jDxh7ctgGy/3P4rYWQRVJD4/Yg= +golang.org/x/net v0.0.0-20220726230323-06994584191e/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220727055044-e65921a090b8 h1:dyU22nBWzrmTQxtNrr4dzVOvaw35nUYE279vF9UmsI8= +golang.org/x/sys v0.0.0-20220727055044-e65921a090b8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/badgerdb-sampler/go_v3.mod b/badgerdb-sampler/go_v3.mod new file mode 100644 index 0000000..c3044af --- /dev/null +++ b/badgerdb-sampler/go_v3.mod @@ -0,0 +1,27 @@ +module github.com/oasisprotocol/badgerdb-sampler + +go 1.25.4 + +require ( + github.com/dgraph-io/badger/v3 v3.2103.5 + github.com/gogo/protobuf v1.3.2 + github.com/tendermint/tendermint v0.34.21 +) + +require ( + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/flatbuffers v1.12.1 // indirect + github.com/klauspost/compress v1.15.9 // indirect + github.com/pkg/errors v0.9.1 // indirect + go.opencensus.io v0.23.0 // indirect + golang.org/x/net v0.0.0-20220726230323-06994584191e // indirect + golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect + google.golang.org/protobuf v1.28.0 // indirect +) diff --git a/badgerdb-sampler/go_v3.sum b/badgerdb-sampler/go_v3.sum new file mode 100644 index 0000000..4ffd7ff --- /dev/null +++ b/badgerdb-sampler/go_v3.sum @@ -0,0 +1,197 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= +github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/tendermint/tendermint v0.34.21 h1:UiGGnBFHVrZhoQVQ7EfwSOLuCtarqCSsRf8VrklqB7s= +github.com/tendermint/tendermint v0.34.21/go.mod h1:XDvfg6U7grcFTDx7VkzxnhazQ/bspGJAn4DZ6DcLLjQ= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20220726230323-06994584191e h1:wOQNKh1uuDGRnmgF0jDxh7ctgGy/3P4rYWQRVJD4/Yg= +golang.org/x/net v0.0.0-20220726230323-06994584191e/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/badgerdb-sampler/go_v4.mod b/badgerdb-sampler/go_v4.mod new file mode 100644 index 0000000..f9332c6 --- /dev/null +++ b/badgerdb-sampler/go_v4.mod @@ -0,0 +1,12 @@ +module github.com/oasisprotocol/badgerdb-sampler + +go 1.25.4 + +require ( + github.com/dgraph-io/badger/v4 v4.2.0 + github.com/gogo/protobuf v1.3.2 + github.com/tendermint/tendermint v0.34.21 +) + +// Note: Dependencies will be filled in when v4 support is implemented +// Run 'go mod tidy' with -modfile=go.mod.v4 to populate diff --git a/badgerdb-sampler/main.go b/badgerdb-sampler/main.go index 2b77ec6..8033a0d 100644 --- a/badgerdb-sampler/main.go +++ b/badgerdb-sampler/main.go @@ -11,13 +11,14 @@ import ( "strings" "time" - badger "github.com/dgraph-io/badger/v2" - "github.com/dgraph-io/badger/v2/options" "github.com/gogo/protobuf/proto" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" tmstore "github.com/tendermint/tendermint/proto/tendermint/store" ) +// BadgerVersion is set by build tags +var BadgerVersion string + // Sample contains both raw and decoded key/value information type Sample struct { RawKey string `json:"raw_key"` @@ -43,26 +44,24 @@ type DBStats struct { func main() { if len(os.Args) < 3 { - fmt.Fprintf(os.Stderr, "Usage: %s [output-subdir] [output-prefix] [max-samples]\n", os.Args[0]) - fmt.Fprintf(os.Stderr, "Example: %s consensus-blockstore-v2 /path/to/blockstore.badger.db\n", os.Args[0]) - fmt.Fprintf(os.Stderr, "Example: %s runtime-mkvs-v2 /path/to/mkvs_storage.badger.db testnet-20220303 emerald 100\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s [output-prefix] [max-samples]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Database types: consensus-blockstore, consensus-evidence, consensus-mkvs, consensus-state, runtime-mkvs, runtime-history\n") + fmt.Fprintf(os.Stderr, "Example: %s consensus-blockstore /path/to/blockstore.badger.db\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Example: %s runtime-mkvs /path/to/mkvs_storage.badger.db ./outputs/testnet-20220303/emerald- 500\n", os.Args[0]) os.Exit(1) } - dbType := os.Args[1] + // Parse arguments + dbType := normalizeDBType(os.Args[1]) dbPath := os.Args[2] - outputSubdir := "" outputPrefix := "" maxSamples := 200 // default if len(os.Args) >= 4 { - outputSubdir = os.Args[3] + outputPrefix = os.Args[3] } if len(os.Args) >= 5 { - outputPrefix = os.Args[4] - } - if len(os.Args) >= 6 { var err error - maxSamples, err = strconv.Atoi(os.Args[5]) + maxSamples, err = strconv.Atoi(os.Args[4]) if err != nil || maxSamples <= 0 { log.Fatalf("Invalid max-samples value: %s (must be positive integer)", os.Args[5]) } @@ -84,7 +83,7 @@ func main() { stats := DBStats{ DatabasePath: dbPath, DatabaseType: dbType, - BadgerDBVersion: "v2.2007.2", + BadgerDBVersion: BadgerVersion, DatabaseSize: dbSize, KeyTypeCounts: make(map[string]int), Samples: []Sample{}, @@ -105,14 +104,13 @@ func main() { fmt.Println(string(output)) // Save results in JSON files in per-snapshot dir and per-database type with optional prefix - if outputSubdir != "" { - outputDir := filepath.Join("/workspace/sampler-outputs", outputSubdir) + if outputPrefix != "" { + outputDir := filepath.Dir(outputPrefix) err = os.MkdirAll(outputDir, 0755) if err != nil { log.Printf("Warning: Could not create directory %s: %v", outputDir, err) } else { - filename := outputPrefix + dbType + ".json" - outputFile := filepath.Join(outputDir, filename) + outputFile := outputPrefix + dbType + ".json" err = os.WriteFile(outputFile, output, 0644) if err != nil { log.Printf("Warning: Could not write to %s: %v", outputFile, err) @@ -123,6 +121,17 @@ func main() { } } +// normalizeDBType removes version suffix from database type (e.g., "consensus-blockstore-v2" -> "consensus-blockstore") +func normalizeDBType(dbType string) string { + // Remove -v2, -v3, -v4 suffix if present + for _, suffix := range []string{"-v2", "-v3", "-v4"} { + if strings.HasSuffix(dbType, suffix) { + return strings.TrimSuffix(dbType, suffix) + } + } + return dbType +} + // getDatabaseSize calculates the total size of all files in the database directory func getDatabaseSize(dbPath string) (int64, error) { var totalSize int64 @@ -138,66 +147,18 @@ func getDatabaseSize(dbPath string) (int64, error) { return totalSize, err } -// openDatabase tries to open the database in ReadOnly mode first, -// then falls back to FileIO mode for corrupted/improperly-closed databases -func openDatabase(path string) (*badger.DB, error) { - // Try ReadOnly mode first (clean databases) - opts := badger.DefaultOptions(path) - opts.ReadOnly = true - opts.Logger = nil - - db, err := badger.Open(opts) - if err == nil { - fmt.Fprintf(os.Stderr, "Opened database in ReadOnly mode\n") - return db, nil - } - - // Check for BadgerDB v3 incompatibility - if strings.Contains(err.Error(), "manifest has unsupported version") { - return nil, fmt.Errorf("database uses BadgerDB v3 (incompatible). Please use sampler-v3 instead.\nOriginal error: %v", err) - } - - fmt.Fprintf(os.Stderr, "ReadOnly mode failed, trying FileIO workaround...\n") - - // FileIO workaround mode - for improperly closed databases - opts = badger.DefaultOptions(path) - opts.ReadOnly = false // Must use read-write for improperly closed DB - opts.SyncWrites = false // Don't sync writes - opts.NumMemtables = 5 // More memtables to avoid flushing - opts.NumLevelZeroTables = 100 // High threshold to avoid compaction - opts.NumLevelZeroTablesStall = 200 // Very high stall threshold - opts.NumCompactors = 0 // Disable background compaction - opts.CompactL0OnClose = false // Don't compact on close - opts.ValueLogLoadingMode = options.FileIO // Use FileIO instead of mmap - opts.TableLoadingMode = options.FileIO // Use FileIO instead of mmap - opts.Logger = nil - - db, err = badger.Open(opts) - if err != nil { - // Check again for v3 incompatibility in FileIO mode - if strings.Contains(err.Error(), "manifest has unsupported version") { - return nil, fmt.Errorf("database uses BadgerDB v3 (incompatible). Please use sampler-v3 instead.\nOriginal error: %v", err) - } - return nil, fmt.Errorf("failed to open database with both ReadOnly and FileIO modes: %v", err) - } - - fmt.Fprintf(os.Stderr, "Opened database in FileIO mode\n") - return db, nil -} - // collectSamples iterates through keys and collects samples with full decoding -func collectSamples(db *badger.DB, stats *DBStats, maxSamples int) error { - return db.View(func(txn *badger.Txn) error { - opts := badger.DefaultIteratorOptions - opts.PrefetchValues = true // Fetch values for sampling - opts.PrefetchSize = maxSamples // Only prefetch what we need +func collectSamples(db *DB, stats *DBStats, maxSamples int) error { + return db.View(func(txn *Txn) error { + opts := DefaultIteratorOptions + opts.PrefetchValues = true + opts.PrefetchSize = maxSamples it := txn.NewIterator(opts) defer it.Close() sampleCount := 0 for it.Rewind(); it.Valid(); it.Next() { - // Only collect samples up to maxSamples if sampleCount >= maxSamples { break } @@ -213,28 +174,28 @@ func collectSamples(db *badger.DB, stats *DBStats, maxSamples int) error { // Analyze key based on database type switch stats.DatabaseType { - case "consensus-blockstore-v2": - keyType, decodedKey := analyzeKeyConsensusBlockstoreV2(key) + case "consensus-blockstore": + keyType, decodedKey := analyzeKeyConsensusBlockstore(key) sample.KeyType = keyType sample.DecodedKey = decodedKey - case "consensus-evidence-v2": - keyType, decodedKey := analyzeKeyConsensusEvidenceV2(key) + case "consensus-evidence": + keyType, decodedKey := analyzeKeyConsensusEvidence(key) sample.KeyType = keyType sample.DecodedKey = decodedKey - case "consensus-mkvs-v2": - keyType, decodedKey := analyzeKeyConsensusMkvsV2(key) + case "consensus-mkvs": + keyType, decodedKey := analyzeKeyConsensusMkvs(key) sample.KeyType = keyType sample.DecodedKey = decodedKey - case "consensus-state-v2": - keyType, decodedKey := analyzeKeyConsensusStateV2(key) + case "consensus-state": + keyType, decodedKey := analyzeKeyConsensusState(key) sample.KeyType = keyType sample.DecodedKey = decodedKey - case "runtime-mkvs-v2": - keyType, decodedKey := analyzeKeyRuntimeMkvsV2(key) + case "runtime-mkvs": + keyType, decodedKey := analyzeKeyRuntimeMkvs(key) sample.KeyType = keyType sample.DecodedKey = decodedKey - case "runtime-history-v2": - keyType, decodedKey := analyzeKeyRuntimeHistoryV2(key) + case "runtime-history": + keyType, decodedKey := analyzeKeyRuntimeHistory(key) sample.KeyType = keyType sample.DecodedKey = decodedKey } @@ -247,20 +208,20 @@ func collectSamples(db *badger.DB, stats *DBStats, maxSamples int) error { // Decode value based on database type switch stats.DatabaseType { - case "consensus-blockstore-v2": - decoded, ts := decodeValueConsensusBlockstoreV2(sample.KeyType, val) + case "consensus-blockstore": + decoded, ts := decodeValueConsensusBlockstore(sample.KeyType, val) sample.DecodedValue = decoded sample.Timestamp = ts - case "consensus-evidence-v2": - sample.DecodedValue = decodeValueConsensusEvidenceV2(sample.KeyType, val) - case "consensus-mkvs-v2": - sample.DecodedValue = decodeValueConsensusMkvsV2(sample.KeyType, val) - case "consensus-state-v2": - sample.DecodedValue = decodeValueConsensusStateV2(sample.KeyType, val) - case "runtime-mkvs-v2": - sample.DecodedValue = decodeValueRuntimeMkvsV2(sample.KeyType, val) - case "runtime-history-v2": - sample.DecodedValue = decodeValueRuntimeHistoryV2(sample.KeyType, val) + case "consensus-evidence": + sample.DecodedValue = decodeValueConsensusEvidence(sample.KeyType, val) + case "consensus-mkvs": + sample.DecodedValue = decodeValueConsensusMkvs(sample.KeyType, val) + case "consensus-state": + sample.DecodedValue = decodeValueConsensusState(sample.KeyType, val) + case "runtime-mkvs": + sample.DecodedValue = decodeValueRuntimeMkvs(sample.KeyType, val) + case "runtime-history": + sample.DecodedValue = decodeValueRuntimeHistory(sample.KeyType, val) } return nil }) @@ -278,8 +239,8 @@ func collectSamples(db *badger.DB, stats *DBStats, maxSamples int) error { }) } -// analyzeKeyConsensusBlockstoreV2 parses consensus-blockstore-v2 key and returns key type and decoded representation -func analyzeKeyConsensusBlockstoreV2(key []byte) (string, string) { +// analyzeKeyConsensusBlockstore parses consensus-blockstore key and returns key type and decoded representation +func analyzeKeyConsensusBlockstore(key []byte) (string, string) { if len(key) < 2 { return "unknown", fmt.Sprintf("%x", key) } @@ -317,9 +278,9 @@ func analyzeKeyConsensusBlockstoreV2(key []byte) (string, string) { return "unknown", asciiKey } -// decodeValueConsensusBlockstoreV2 attempts to decode protobuf value based on key type +// decodeValueConsensusBlockstore attempts to decode protobuf value based on key type // Returns: (decoded_string, timestamp_unix) -func decodeValueConsensusBlockstoreV2(keyType string, value []byte) (string, int64) { +func decodeValueConsensusBlockstore(keyType string, value []byte) (string, int64) { switch keyType { case "blockstore_state": var state tmstore.BlockStoreState @@ -368,7 +329,6 @@ func decodeValueConsensusBlockstoreV2(keyType string, value []byte) (string, int case "block_commit": var commit tmproto.Commit if err := proto.Unmarshal(value, &commit); err == nil { - // Commits don't have a direct timestamp field; timestamp comes from block header return fmt.Sprintf("Commit{height: %d, round: %d, signatures: %d}", commit.Height, commit.Round, len(commit.Signatures)), 0 } @@ -377,7 +337,6 @@ func decodeValueConsensusBlockstoreV2(keyType string, value []byte) (string, int case "seen_commit": var commit tmproto.Commit if err := proto.Unmarshal(value, &commit); err == nil { - // Commits don't have a direct timestamp field; timestamp comes from block header return fmt.Sprintf("SeenCommit{height: %d, round: %d, signatures: %d}", commit.Height, commit.Round, len(commit.Signatures)), 0 } @@ -396,8 +355,8 @@ func decodeValueConsensusBlockstoreV2(keyType string, value []byte) (string, int } } -// analyzeKeyConsensusEvidenceV2 parses consensus-evidence-v2 key and returns key type and decoded representation -func analyzeKeyConsensusEvidenceV2(key []byte) (string, string) { +// analyzeKeyConsensusEvidence parses consensus-evidence key and returns key type and decoded representation +func analyzeKeyConsensusEvidence(key []byte) (string, string) { if len(key) < 2 { return "unknown", fmt.Sprintf("%x", key) } @@ -408,14 +367,11 @@ func analyzeKeyConsensusEvidenceV2(key []byte) (string, string) { } // Evidence DB is typically empty or uses simple key patterns - // Return the hex representation for now return fmt.Sprintf("type_%02x", key[1]), fmt.Sprintf("%x", key[1:]) } -// decodeValueConsensusEvidenceV2 attempts to decode evidence value -func decodeValueConsensusEvidenceV2(keyType string, value []byte) string { - // Evidence values are protobuf-encoded DuplicateVoteEvidence or LightClientAttackEvidence - // For now, just show size as evidence DB is typically empty +// decodeValueConsensusEvidence attempts to decode evidence value +func decodeValueConsensusEvidence(keyType string, value []byte) string { if len(value) == 0 { return "Empty" } @@ -430,40 +386,44 @@ func decodeValueConsensusEvidenceV2(keyType string, value []byte) string { return fmt.Sprintf("Evidence (size: %d bytes, type: %s)", len(value), keyType) } -// analyzeKeyConsensusMkvsV2 parses consensus-mkvs-v2 key and returns key type and decoded representation -// MKVS keys: 0xNN (type prefix) + type-specific data (NO dbVersion prefix) -// Key formats from oasis-core/go/storage/mkvs/db/badger/badger.go: -// 0x00: node (hash) -// 0x01: writeLog (version uint64, new root, old root) -// 0x02: rootsMetadata (version uint64) -// 0x03: rootUpdatedNodes (version uint64, root) -// 0x04: metadata -// 0x05: multipartRestoreNodeLog (hash) -// 0x06: rootNode (typed hash) -func analyzeKeyConsensusMkvsV2(key []byte) (string, string) { +// analyzeKeyConsensusMkvs parses consensus-mkvs key and returns key type and decoded representation +// FIXED: Handles both old format (no dbVersion prefix) and new format (0x01 or 0x05 dbVersion prefix) +// MKVS keys from oasis-core/go/storage/mkvs/db/badger/badger.go +func analyzeKeyConsensusMkvs(key []byte) (string, string) { if len(key) < 1 { return "unknown", fmt.Sprintf("%x", key) } - // MKVS keys start directly with the type prefix byte (0x00-0x06) - // NO dbVersion prefix for MKVS storage + // Check if key has dbVersion prefix (0x01 or 0x05) + // Old format: type byte directly at key[0] + // New format: 0x01 or 0x05 at key[0], type byte at key[1] prefixByte := key[0] - data := key[1:] + var data []byte + + if prefixByte == 0x01 || prefixByte == 0x05 { + // New format with dbVersion prefix + if len(key) < 2 { + return "unknown", fmt.Sprintf("%x", key) + } + prefixByte = key[1] + data = key[2:] + } else { + // Old format without dbVersion prefix (BadgerDB v2 MKVS) + data = key[1:] + } switch prefixByte { case 0x00: // nodeKeyFmt: hash.Hash (32 bytes) - if len(data) >= 32 { - return "node", fmt.Sprintf("node:hash=%x", data[:32]) - } - return "node", fmt.Sprintf("node:hash=%x", data) + return "node", fmt.Sprintf("node:hash=%x", truncateBytes(data, 8)) case 0x01: // writeLogKeyFmt: uint64(version) + TypedHash(new root) + TypedHash(old root) if len(data) >= 8 { version := binary.BigEndian.Uint64(data[0:8]) - if len(data) >= 41 { - return "write_log", fmt.Sprintf("write_log:v=%d,new_root=%x,old_root=%x", - version, truncateBytes(data[8:41], 8), truncateBytes(data[41:], 8)) + if len(data) >= 8+33 { // version + first TypedHash (33 bytes in v3+) + rootType := data[8] + rootHash := data[9:9+32] + return "write_log", fmt.Sprintf("write_log:v=%d,new_root_type=%d,hash=%x...", version, rootType, truncateBytes(rootHash, 4)) } return "write_log", fmt.Sprintf("write_log:v=%d", version) } @@ -479,9 +439,10 @@ func analyzeKeyConsensusMkvsV2(key []byte) (string, string) { // rootUpdatedNodesKeyFmt: uint64(version) + TypedHash(root) if len(data) >= 8 { version := binary.BigEndian.Uint64(data[0:8]) - if len(data) >= 41 { - return "root_updated_nodes", fmt.Sprintf("root_updated_nodes:v=%d,root=%x", - version, truncateBytes(data[8:41], 8)) + if len(data) >= 8+33 { // version + TypedHash + rootType := data[8] + rootHash := data[9:9+32] + return "root_updated_nodes", fmt.Sprintf("root_updated_nodes:v=%d,root_type=%d,hash=%x...", version, rootType, truncateBytes(rootHash, 4)) } return "root_updated_nodes", fmt.Sprintf("root_updated_nodes:v=%d", version) } @@ -490,36 +451,32 @@ func analyzeKeyConsensusMkvsV2(key []byte) (string, string) { // metadataKeyFmt: no additional data return "metadata", "metadata" case 0x05: - // multipartRestoreNodeLogKeyFmt: TypedHash (33 bytes: type byte + 32-byte hash) + // multipartRestoreNodeLogKeyFmt: TypedHash (33 bytes) if len(data) >= 33 { - return "multipart_restore_log", fmt.Sprintf("multipart_restore_log:hash=%x", data[:33]) + rootType := data[0] + rootHash := data[1:33] + return "multipart_restore_log", fmt.Sprintf("multipart_restore_log:type=%d,hash=%x", rootType, truncateBytes(rootHash, 8)) } - return "multipart_restore_log", fmt.Sprintf("multipart_restore_log:hash=%x", data) + return "multipart_restore_log", fmt.Sprintf("multipart_restore_log:hash=%x", truncateBytes(data, 8)) case 0x06: - // rootNodeKeyFmt: TypedHash (33 bytes: type byte + 32-byte hash) + // rootNodeKeyFmt: TypedHash (33 bytes) if len(data) >= 33 { - return "root_node", fmt.Sprintf("root_node:hash=%x", data[:33]) + rootType := data[0] + rootHash := data[1:33] + return "root_node", fmt.Sprintf("root_node:type=%d,hash=%x", rootType, truncateBytes(rootHash, 8)) } - return "root_node", fmt.Sprintf("root_node:hash=%x", data) + return "root_node", fmt.Sprintf("root_node:hash=%x", truncateBytes(data, 8)) default: return fmt.Sprintf("unknown_%02x", prefixByte), fmt.Sprintf("unknown_%02x:%x", prefixByte, truncateBytes(data, 8)) } } -// decodeValueConsensusMkvsV2 decodes MKVS value -// Value types: -// - node: Serialized node data (binary) -// - write_log: CBOR-serialized write log -// - roots_metadata: CBOR-serialized rootsMetadata -// - root_updated_nodes: CBOR-serialized []updatedNode -// - metadata: CBOR-serialized metadata -func decodeValueConsensusMkvsV2(keyType string, value []byte) string { +// decodeValueConsensusMkvs decodes MKVS value +func decodeValueConsensusMkvs(keyType string, value []byte) string { if len(value) == 0 { return "Empty" } - // For CBOR-encoded values, we could decode but it's complex - // Just report size and encoding type switch keyType { case "node": return fmt.Sprintf("Serialized node (size: %d bytes)", len(value)) @@ -530,9 +487,8 @@ func decodeValueConsensusMkvsV2(keyType string, value []byte) string { } } -// analyzeKeyConsensusStateV2 parses consensus-state-v2 key and returns key type and decoded representation -// State keys are ASCII strings: 0x01 + ":" -func analyzeKeyConsensusStateV2(key []byte) (string, string) { +// analyzeKeyConsensusState parses consensus-state key and returns key type and decoded representation +func analyzeKeyConsensusState(key []byte) (string, string) { if len(key) < 2 { return "unknown", fmt.Sprintf("%x", key) } @@ -573,154 +529,34 @@ func analyzeKeyConsensusStateV2(key []byte) (string, string) { return "text_key", decoded } -// decodeValueConsensusStateV2 decodes state value (protobuf-encoded Tendermint state data) -func decodeValueConsensusStateV2(keyType string, value []byte) string { - // State values are protobuf-encoded, but the exact message type depends on the key - // For simplicity, just show size and type info +// decodeValueConsensusState decodes state value +func decodeValueConsensusState(keyType string, value []byte) string { if len(value) == 0 { return "Empty" } - return fmt.Sprintf("Tendermint %s data (size: %d bytes)", keyType, len(value)) } -// analyzeKeyRuntimeMkvsV2 parses runtime-mkvs-v2 key and returns key type and decoded representation -// MKVS keys: 0xNN (type prefix) + type-specific data (NO dbVersion prefix) -// Same format as consensus-mkvs-v2 from oasis-core/go/storage/mkvs/db/badger/badger.go -func analyzeKeyRuntimeMkvsV2(key []byte) (string, string) { - if len(key) < 1 { - return "unknown", fmt.Sprintf("%x", key) - } - - // MKVS keys start directly with the type prefix byte (0x00-0x06) - // NO dbVersion prefix for MKVS storage - prefixByte := key[0] - data := key[1:] - - switch prefixByte { - case 0x00: - // nodeKeyFmt: hash.Hash (32 bytes) - if len(data) >= 32 { - return "node", fmt.Sprintf("node:hash=%x", data[:32]) - } - return "node", fmt.Sprintf("node:hash=%x", data) - case 0x01: - // writeLogKeyFmt: uint64(version) + TypedHash(new root) + TypedHash(old root) - if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - if len(data) >= 41 { - return "write_log", fmt.Sprintf("write_log:v=%d,new_root=%x,old_root=%x", - version, truncateBytes(data[8:41], 8), truncateBytes(data[41:], 8)) - } - return "write_log", fmt.Sprintf("write_log:v=%d", version) - } - return "write_log", "write_log:" - case 0x02: - // rootsMetadataKeyFmt: uint64(version) - if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - return "roots_metadata", fmt.Sprintf("roots_metadata:v=%d", version) - } - return "roots_metadata", "roots_metadata:" - case 0x03: - // rootUpdatedNodesKeyFmt: uint64(version) + TypedHash(root) - if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - if len(data) >= 41 { - return "root_updated_nodes", fmt.Sprintf("root_updated_nodes:v=%d,root=%x", - version, truncateBytes(data[8:41], 8)) - } - return "root_updated_nodes", fmt.Sprintf("root_updated_nodes:v=%d", version) - } - return "root_updated_nodes", "root_updated_nodes:" - case 0x04: - // metadataKeyFmt: no additional data - return "metadata", "metadata" - case 0x05: - // multipartRestoreNodeLogKeyFmt: TypedHash (33 bytes: type byte + 32-byte hash) - if len(data) >= 33 { - return "multipart_restore_log", fmt.Sprintf("multipart_restore_log:hash=%x", data[:33]) - } - return "multipart_restore_log", fmt.Sprintf("multipart_restore_log:hash=%x", data) - case 0x06: - // rootNodeKeyFmt: TypedHash (33 bytes: type byte + 32-byte hash) - if len(data) >= 33 { - return "root_node", fmt.Sprintf("root_node:hash=%x", data[:33]) - } - return "root_node", fmt.Sprintf("root_node:hash=%x", data) - default: - return fmt.Sprintf("unknown_%02x", prefixByte), fmt.Sprintf("unknown_%02x:%x", prefixByte, truncateBytes(data, 8)) - } +// analyzeKeyRuntimeMkvs parses runtime-mkvs key (same format as consensus-mkvs) +func analyzeKeyRuntimeMkvs(key []byte) (string, string) { + return analyzeKeyConsensusMkvs(key) } -// decodeValueRuntimeMkvsV2 decodes runtime MKVS value -// Same format as consensus MKVS (CBOR-encoded for most types) -func decodeValueRuntimeMkvsV2(keyType string, value []byte) string { - if len(value) == 0 { - return "Empty" - } - - switch keyType { - case "node": - return fmt.Sprintf("Serialized node (size: %d bytes)", len(value)) - case "write_log", "roots_metadata", "root_updated_nodes", "metadata": - return fmt.Sprintf("CBOR-encoded %s (size: %d bytes)", keyType, len(value)) - default: - return fmt.Sprintf("MKVS %s data (size: %d bytes)", keyType, len(value)) - } +// decodeValueRuntimeMkvs decodes runtime MKVS value (same format as consensus MKVS) +func decodeValueRuntimeMkvs(keyType string, value []byte) string { + return decodeValueConsensusMkvs(keyType, value) } -// analyzeKeyRuntimeHistoryV2 parses runtime-history-v2 key and returns key type and decoded representation -// History keys are ASCII strings: 0x01 + ":" -func analyzeKeyRuntimeHistoryV2(key []byte) (string, string) { - if len(key) < 2 { - return "unknown", fmt.Sprintf("%x", key) - } - - // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) - if key[0] != 0x01 { - return "unknown", fmt.Sprintf("%x", key) - } - - // Parse ASCII key after 0x01 prefix - decoded := string(key[1:]) - - // Check if it's printable ASCII - if !isPrintableASCII(decoded) { - return "binary", fmt.Sprintf("%x", key) - } - - // Extract key type prefix before colon - colonIdx := strings.IndexByte(decoded, ':') - if colonIdx != -1 { - prefix := decoded[:colonIdx] - switch prefix { - case "abciResponsesKey": - return "abci_responses", decoded - case "consensusParamsKey": - return "consensus_params", decoded - case "validatorsKey": - return "validators", decoded - case "stateKey": - return "state", decoded - case "genesisDoc": - return "genesis", decoded - default: - return prefix, decoded - } - } - - return "text_key", decoded +// analyzeKeyRuntimeHistory parses runtime-history key (same format as consensus-state) +func analyzeKeyRuntimeHistory(key []byte) (string, string) { + return analyzeKeyConsensusState(key) } -// decodeValueRuntimeHistoryV2 decodes runtime history value -func decodeValueRuntimeHistoryV2(keyType string, value []byte) string { - // Runtime history values are similar to consensus state - // For simplicity, just show size and type info +// decodeValueRuntimeHistory decodes runtime history value +func decodeValueRuntimeHistory(keyType string, value []byte) string { if len(value) == 0 { return "Empty" } - return fmt.Sprintf("Runtime %s data (size: %d bytes)", keyType, len(value)) } From 91c34a72a48e24535d489a8c518695aa5e0cd19a Mon Sep 17 00:00:00 2001 From: gw0 Date: Tue, 18 Nov 2025 12:37:52 +0100 Subject: [PATCH 03/22] badgerdb-sampler: Add helper scripts --- badgerdb-sampler/scripts/download-mainnet.sh | 29 ++++++++ badgerdb-sampler/scripts/download-testnet.sh | 33 +++++++++ badgerdb-sampler/scripts/mount-snapshots.sh | 23 ++++++ .../scripts/run-mainnet-samplers.sh | 62 ++++++++++++++++ .../scripts/run-testnet-samplers.sh | 74 +++++++++++++++++++ badgerdb-sampler/scripts/umount-snapshots.sh | 20 +++++ 6 files changed, 241 insertions(+) create mode 100755 badgerdb-sampler/scripts/download-mainnet.sh create mode 100755 badgerdb-sampler/scripts/download-testnet.sh create mode 100755 badgerdb-sampler/scripts/mount-snapshots.sh create mode 100755 badgerdb-sampler/scripts/run-mainnet-samplers.sh create mode 100755 badgerdb-sampler/scripts/run-testnet-samplers.sh create mode 100755 badgerdb-sampler/scripts/umount-snapshots.sh diff --git a/badgerdb-sampler/scripts/download-mainnet.sh b/badgerdb-sampler/scripts/download-mainnet.sh new file mode 100755 index 0000000..7f0fc8a --- /dev/null +++ b/badgerdb-sampler/scripts/download-mainnet.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Script to download Oasis node snapshots for Mainnet +set -euo pipefail + +curl -Lo mainnet/20201001-20201118/README.md https://snapshots.oasis.io/node/mainnet/20201001-20201118/README.md +curl -Lo mainnet/20201001-20201118/consensus.tar.zst.list https://snapshots.oasis.io/node/mainnet/20201001-20201118/consensus.tar.zst.list +curl -Lo mainnet/20201001-20201118/consensus.tar.zst https://snapshots.oasis.io/node/mainnet/20201001-20201118/consensus.tar.zst + +curl -Lo mainnet/20201118-20210428/README.md https://snapshots.oasis.io/node/mainnet/20201118-20210428/README.md +curl -Lo mainnet/20201118-20210428/consensus.tar.zst.list https://snapshots.oasis.io/node/mainnet/20201118-20210428/consensus.tar.zst.list +curl -Lo mainnet/20201118-20210428/consensus.tar.zst https://snapshots.oasis.io/node/mainnet/20201118-20210428/consensus.tar.zst + +curl -Lo mainnet/20210428-20220411/README.md https://snapshots.oasis.io/node/mainnet/20210428-20220411/README.md +curl -Lo mainnet/20210428-20220411/consensus.tar.zst.list https://snapshots.oasis.io/node/mainnet/20210428-20220411/consensus.tar.zst.list +curl -Lo mainnet/20210428-20220411/consensus.tar.zst https://snapshots.oasis.io/node/mainnet/20210428-20220411/consensus.tar.zst +curl -Lo mainnet/20210428-20220411/cipher.tar.zst.list https://snapshots.oasis.io/node/mainnet/20210428-20220411/cipher.tar.zst.list +curl -Lo mainnet/20210428-20220411/cipher.tar.zst https://snapshots.oasis.io/node/mainnet/20210428-20220411/cipher.tar.zst +curl -Lo mainnet/20210428-20220411/emerald.tar.zst.list https://snapshots.oasis.io/node/mainnet/20210428-20220411/emerald.tar.zst.list +curl -Lo mainnet/20210428-20220411/emerald.tar.zst https://snapshots.oasis.io/node/mainnet/20210428-20220411/emerald.tar.zst + +curl -Lo mainnet/20220411-20231129/README.md https://snapshots.oasis.io/node/mainnet/20220411-20231129/README.md +curl -Lo mainnet/20220411-20231129/consensus.tar.zst.list https://snapshots.oasis.io/node/mainnet/20220411-20231129/consensus.tar.zst.list +curl -Lo mainnet/20220411-20231129/consensus.tar.zst https://snapshots.oasis.io/node/mainnet/20220411-20231129/consensus.tar.zst +curl -Lo mainnet/20220411-20231129/cipher.tar.zst.list https://snapshots.oasis.io/node/mainnet/20220411-20231129/cipher.tar.zst.list +curl -Lo mainnet/20220411-20231129/cipher.tar.zst https://snapshots.oasis.io/node/mainnet/20220411-20231129/cipher.tar.zst +curl -Lo mainnet/20220411-20231129/emerald.tar.zst.list https://snapshots.oasis.io/node/mainnet/20220411-20231129/emerald.tar.zst.list +curl -Lo mainnet/20220411-20231129/emerald.tar.zst https://snapshots.oasis.io/node/mainnet/20220411-20231129/emerald.tar.zst +curl -Lo mainnet/20220411-20231129/sapphire.tar.zst.list https://snapshots.oasis.io/node/mainnet/20220411-20231129/sapphire.tar.zst.list +curl -Lo mainnet/20220411-20231129/sapphire.tar.zst https://snapshots.oasis.io/node/mainnet/20220411-20231129/sapphire.tar.zst diff --git a/badgerdb-sampler/scripts/download-testnet.sh b/badgerdb-sampler/scripts/download-testnet.sh new file mode 100755 index 0000000..1e31bcb --- /dev/null +++ b/badgerdb-sampler/scripts/download-testnet.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Script to download Oasis node snapshots for Testnet +set -euo pipefail + +curl -Lo testnet/20200915-20201104/README.md https://snapshots.oasis.io/node/testnet/20200915-20201104/README.md +curl -Lo testnet/20200915-20201104/consensus_testnet.tar.zst.list https://snapshots.oasis.io/node/testnet/20200915-20201104/consensus_testnet.tar.zst.list +curl -Lo testnet/20200915-20201104/consensus_testnet.tar.zst https://snapshots.oasis.io/node/testnet/20200915-20201104/consensus_testnet.tar.zst + +curl -Lo testnet/20201104-20210203/README.md https://snapshots.oasis.io/node/testnet/20201104-20210203/README.md +curl -Lo testnet/20201104-20210203/consensus_testnet.tar.zst.list https://snapshots.oasis.io/node/testnet/20201104-20210203/consensus_testnet.tar.zst.list +curl -Lo testnet/20201104-20210203/consensus_testnet.tar.zst https://snapshots.oasis.io/node/testnet/20201104-20210203/consensus_testnet.tar.zst + +curl -Lo testnet/20210203-20210324/README.md https://snapshots.oasis.io/node/testnet/20210203-20210324/README.md +curl -Lo testnet/20210203-20210324/consensus_testnet.tar.zst.list https://snapshots.oasis.io/node/testnet/20210203-20210324/consensus_testnet.tar.zst.list +curl -Lo testnet/20210203-20210324/consensus_testnet.tar.zst https://snapshots.oasis.io/node/testnet/20210203-20210324/consensus_testnet.tar.zst + +curl -Lo testnet/20210324-20210413/README.md https://snapshots.oasis.io/node/testnet/20210324-20210413/README.md +curl -Lo testnet/20210324-20210413/consensus_testnet.tar.zst.list https://snapshots.oasis.io/node/testnet/20210324-20210413/consensus_testnet.tar.zst.list +curl -Lo testnet/20210324-20210413/consensus_testnet.tar.zst https://snapshots.oasis.io/node/testnet/20210324-20210413/consensus_testnet.tar.zst + +curl -Lo testnet/20210413-20220303/README.md https://snapshots.oasis.io/node/testnet/20210413-20220303/README.md +curl -Lo testnet/20210413-20220303/consensus_testnet.tar.zst.list https://snapshots.oasis.io/node/testnet/20210413-20220303/consensus_testnet.tar.zst.list +curl -Lo testnet/20210413-20220303/consensus_testnet.tar.zst https://snapshots.oasis.io/node/testnet/20210413-20220303/consensus_testnet.tar.zst + +curl -Lo testnet/20220303-20231012/README.md https://snapshots.oasis.io/node/testnet/20220303-20231012/README.md +curl -Lo testnet/20220303-20231012/consensus_testnet.tar.zst.list https://snapshots.oasis.io/node/testnet/20220303-20231012/consensus_testnet.tar.zst.list +curl -Lo testnet/20220303-20231012/consensus_testnet.tar.zst https://snapshots.oasis.io/node/testnet/20220303-20231012/consensus_testnet.tar.zst +curl -Lo testnet/20220303-20231012/cipher_testnet.tar.zst.list https://snapshots.oasis.io/node/testnet/20220303-20231012/cipher_testnet.tar.zst.list +curl -Lo testnet/20220303-20231012/cipher_testnet.tar.zst https://snapshots.oasis.io/node/testnet/20220303-20231012/cipher_testnet.tar.zst +curl -Lo testnet/20220303-20231012/emerald_testnet.tar.zst.list https://snapshots.oasis.io/node/testnet/20220303-20231012/emerald_testnet.tar.zst.list +curl -Lo testnet/20220303-20231012/emerald_testnet.tar.zst https://snapshots.oasis.io/node/testnet/20220303-20231012/emerald_testnet.tar.zst +curl -Lo testnet/20220303-20231012/sapphire_testnet.tar.zst.list https://snapshots.oasis.io/node/testnet/20220303-20231012/sapphire_testnet.tar.zst.list +curl -Lo testnet/20220303-20231012/sapphire_testnet.tar.zst https://snapshots.oasis.io/node/testnet/20220303-20231012/sapphire_testnet.tar.zst diff --git a/badgerdb-sampler/scripts/mount-snapshots.sh b/badgerdb-sampler/scripts/mount-snapshots.sh new file mode 100755 index 0000000..49a1ef8 --- /dev/null +++ b/badgerdb-sampler/scripts/mount-snapshots.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Script to mount .tar.zst snapshot archives without extracting +set -euo pipefail + +cd "$(dirname "$0")" +search_dir="${1:-.}" + +find "$search_dir" -name "*.tar.zst" -type f | while read -r archive; do + basename="${archive%.tar.zst}" + mountpoint="${basename}.mount" + overlay="${basename}.overlay" + + if mountpoint -q "$mountpoint" 2>/dev/null; then + echo "Already mounted: $mountpoint" + continue + fi + + echo "Mounting: $archive -> $mountpoint" + mkdir -p "$mountpoint" + ./ratarmount -o allow_other,uid=1000,gid=1000 -w "$overlay" "$archive" "$mountpoint" +done + +echo "Done!" diff --git a/badgerdb-sampler/scripts/run-mainnet-samplers.sh b/badgerdb-sampler/scripts/run-mainnet-samplers.sh new file mode 100755 index 0000000..5ed2b19 --- /dev/null +++ b/badgerdb-sampler/scripts/run-mainnet-samplers.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Script to run all badgerdb-samplers on Mainnet +set -euo pipefail + +SAMPLER_V2="./bin/badgerdb-sampler-v2" +SAMPLER_V3="./bin/badgerdb-sampler-v3" +OUTPUT="./outputs" + +mkdir -p "$OUTPUT" + +# $OUTPUT/mainnet/20201001-20201118/ (BadgerDB v2) +( +mkdir -p "$OUTPUT/mainnet-20201001-20201118" +$SAMPLER_V2 consensus-blockstore-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20201001-20201118/ > $OUTPUT/mainnet-20201001-20201118/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20201001-20201118/ > $OUTPUT/mainnet-20201001-20201118/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20201001-20201118/ > $OUTPUT/mainnet-20201001-20201118/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20201001-20201118/ > $OUTPUT/mainnet-20201001-20201118/consensus-mkvs-v2.log 2>&1 || true +) & + +# $OUTPUT/mainnet/20201118-20210428/ (BadgerDB v2) +( +mkdir -p "$OUTPUT/mainnet-20201118-20210428" +$SAMPLER_V2 consensus-blockstore-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20201118-20210428/ > $OUTPUT/mainnet-20201118-20210428/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20201118-20210428/ > $OUTPUT/mainnet-20201118-20210428/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20201118-20210428/ > $OUTPUT/mainnet-20201118-20210428/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20201118-20210428/ > $OUTPUT/mainnet-20201118-20210428/consensus-mkvs-v2.log 2>&1 || true +) & + +# $OUTPUT/mainnet/20210428-20220411/ (BadgerDB v3) +( +mkdir -p "$OUTPUT/mainnet-20210428-20220411" +$SAMPLER_V3 consensus-blockstore-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20210428-20220411/ > $OUTPUT/mainnet-20210428-20220411/consensus-blockstore-v3.log 2>&1 || true +$SAMPLER_V3 consensus-evidence-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20210428-20220411/ > $OUTPUT/mainnet-20210428-20220411/consensus-evidence-v3.log 2>&1 || true +$SAMPLER_V3 consensus-state-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20210428-20220411/ > $OUTPUT/mainnet-20210428-20220411/consensus-state-v3.log 2>&1 || true +$SAMPLER_V3 consensus-mkvs-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/ > $OUTPUT/mainnet-20210428-20220411/consensus-mkvs-v3.log 2>&1 || true + +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20210428-20220411/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/ cipher > $OUTPUT/mainnet-20210428-20220411/cipher-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20210428-20220411/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/ emerald > $OUTPUT/mainnet-20210428-20220411/emerald-runtime-mkvs-v3.log 2>&1 || true + +$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20210428-20220411/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT/mainnet-20210428-20220411/ cipher > $OUTPUT/mainnet-20210428-20220411/cipher-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20210428-20220411/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT/mainnet-20210428-20220411/ emerald > $OUTPUT/mainnet-20210428-20220411/emerald-runtime-history-v3.log 2>&1 || true +) & + +# $OUTPUT/mainnet/20220411-20231129/ (BadgerDB v3) +( +mkdir -p "$OUTPUT/mainnet-20220411-20231129" +$SAMPLER_V3 consensus-blockstore-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20220411-20231129/ > $OUTPUT/mainnet-20220411-20231129/consensus-blockstore-v3.log 2>&1 || true +$SAMPLER_V3 consensus-evidence-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20220411-20231129/ > $OUTPUT/mainnet-20220411-20231129/consensus-evidence-v3.log 2>&1 || true +$SAMPLER_V3 consensus-state-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20220411-20231129/ > $OUTPUT/mainnet-20220411-20231129/consensus-state-v3.log 2>&1 || true +$SAMPLER_V3 consensus-mkvs-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/ > $OUTPUT/mainnet-20220411-20231129/consensus-mkvs-v3.log 2>&1 || true + +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/cipher- > $OUTPUT/mainnet-20220411-20231129/cipher-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/emerald- > $OUTPUT/mainnet-20220411-20231129/emerald-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/sapphire.mount/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/sapphire- > $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-mkvs-v3.log 2>&1 || true + +$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT/mainnet-20220411-20231129/cipher- > $OUTPUT/mainnet-20220411-20231129/cipher-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT/mainnet-20220411-20231129/emerald- > $OUTPUT/mainnet-20220411-20231129/emerald-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/sapphire.mount/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/history.db $OUTPUT/mainnet-20220411-20231129/sapphire- > $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-history-v3.log 2>&1 || true +) & + +echo "All samplers launched in background!" +ps aux | grep badgerdb-sampler diff --git a/badgerdb-sampler/scripts/run-testnet-samplers.sh b/badgerdb-sampler/scripts/run-testnet-samplers.sh new file mode 100755 index 0000000..79f1811 --- /dev/null +++ b/badgerdb-sampler/scripts/run-testnet-samplers.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Script to run all badgerdb-samplers on Testnet +set -euo pipefail + +SAMPLER_V2="./bin/badgerdb-sampler-v2" +SAMPLER_V3="./bin/badgerdb-sampler-v3" +OUTPUT="./outputs" + +mkdir -p "$OUTPUT" + +# $OUTPUT/testnet/20200915-20201104/ (BadgerDB v2) +( +mkdir -p "$OUTPUT/testnet-20200915-20201104" +$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20200915-20201104/ > $OUTPUT/testnet-20200915-20201104/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20200915-20201104/ > $OUTPUT/testnet-20200915-20201104/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20200915-20201104/ > $OUTPUT/testnet-20200915-20201104/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20200915-20201104/ > $OUTPUT/testnet-20200915-20201104/consensus-mkvs-v2.log 2>&1 || true +) & + +# $OUTPUT/testnet/20201104-20210203/ (BadgerDB v2) +( +mkdir -p "$OUTPUT/testnet-20201104-20210203" +$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20201104-20210203/ > $OUTPUT/testnet-20201104-20210203/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20201104-20210203/ > $OUTPUT/testnet-20201104-20210203/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20201104-20210203/ > $OUTPUT/testnet-20201104-20210203/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20201104-20210203/ > $OUTPUT/testnet-20201104-20210203/consensus-mkvs-v2.log 2>&1 || true +) & + +# $OUTPUT/testnet/20210203-20210324/ (BadgerDB v2) +( +mkdir -p "$OUTPUT/testnet-20210203-20210324" +$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210203-20210324/ > $OUTPUT/testnet-20210203-20210324/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210203-20210324/ > $OUTPUT/testnet-20210203-20210324/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210203-20210324/ > $OUTPUT/testnet-20210203-20210324/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20210203-20210324/ > $OUTPUT/testnet-20210203-20210324/consensus-mkvs-v2.log 2>&1 || true +) & + +# $OUTPUT/testnet/20210324-20210413/ (BadgerDB v2) +( +mkdir -p "$OUTPUT/testnet-20210324-20210413" +$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210324-20210413/ > $OUTPUT/testnet-20210324-20210413/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210324-20210413/ > $OUTPUT/testnet-20210324-20210413/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210324-20210413/ > $OUTPUT/testnet-20210324-20210413/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20210324-20210413/ > $OUTPUT/testnet-20210324-20210413/consensus-mkvs-v2.log 2>&1 || true +) & + +# $OUTPUT/testnet/20210413-20220303/ (BadgerDB v3) +( +mkdir -p "$OUTPUT/testnet-20210413-20220303" +$SAMPLER_V3 consensus-blockstore-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210413-20220303/ > $OUTPUT/testnet-20210413-20220303/consensus-blockstore-v3.log 2>&1 || true +$SAMPLER_V3 consensus-evidence-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210413-20220303/ > $OUTPUT/testnet-20210413-20220303/consensus-evidence-v3.log 2>&1 || true +$SAMPLER_V3 consensus-state-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210413-20220303/ > $OUTPUT/testnet-20210413-20220303/consensus-state-v3.log 2>&1 || true +$SAMPLER_V3 consensus-mkvs-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20210413-20220303/ > $OUTPUT/testnet-20210413-20220303/consensus-mkvs-v3.log 2>&1 || true +) & + +# $OUTPUT/testnet/20220303-20231012/ (BadgerDB v3) +( +mkdir -p "$OUTPUT/testnet-20220303-20231012" +$SAMPLER_V3 consensus-blockstore-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20220303-20231012/ > $OUTPUT/testnet-20220303-20231012/consensus-blockstore-v3.log 2>&1 || true +$SAMPLER_V3 consensus-evidence-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20220303-20231012/ > $OUTPUT/testnet-20220303-20231012/consensus-evidence-v3.log 2>&1 || true +$SAMPLER_V3 consensus-state-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20220303-20231012/ > $OUTPUT/testnet-20220303-20231012/consensus-state-v3.log 2>&1 || true +$SAMPLER_V3 consensus-mkvs-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/ > $OUTPUT/testnet-20220303-20231012/consensus-mkvs-v3.log 2>&1 || true + +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/cipher_testnet.mount/runtimes/0000000000000000000000000000000000000000000000000000000000000000/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/cipher- > $OUTPUT/testnet-20220303-20231012/cipher-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/emerald_testnet.mount/runtimes/00000000000000000000000000000000000000000000000072c8215e60d5bca7/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/emerald- > $OUTPUT/testnet-20220303-20231012/emerald-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/sapphire_testnet.mount/runtimes/000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/sapphire- > $OUTPUT/testnet-20220303-20231012/sapphire-runtime-mkvs-v3.log 2>&1 || true + +$SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/cipher_testnet.mount/runtimes/0000000000000000000000000000000000000000000000000000000000000000/history.db $OUTPUT/testnet-20220303-20231012/cipher- > $OUTPUT/testnet-20220303-20231012/cipher-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/emerald_testnet.mount/runtimes/00000000000000000000000000000000000000000000000072c8215e60d5bca7/history.db $OUTPUT/testnet-20220303-20231012/emerald- > $OUTPUT/testnet-20220303-20231012/emerald-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/sapphire_testnet.mount/runtimes/000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c/history.db $OUTPUT/testnet-20220303-20231012/sapphire- > $OUTPUT/testnet-20220303-20231012/sapphire-runtime-history-v3.log 2>&1 || true +) & + +echo "All samplers launched in background!" +ps aux | grep badgerdb-sampler diff --git a/badgerdb-sampler/scripts/umount-snapshots.sh b/badgerdb-sampler/scripts/umount-snapshots.sh new file mode 100755 index 0000000..f31b0d0 --- /dev/null +++ b/badgerdb-sampler/scripts/umount-snapshots.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Script to unmount .tar.zst snapshot archives and remove overlays +set -euo pipefail + +cd "$(dirname "$0")" +search_dir="${1:-.}" + +find "$search_dir" -name "*.tar.zst" -type f | while read -r archive; do + basename="${archive%.tar.zst}" + mountpoint="${basename}.mount" + overlay="${basename}.overlay" + + if mountpoint -q "$mountpoint" 2>/dev/null; then + echo "Unmounting: $mountpoint" + fusermount -u "$mountpoint" + rm -rf "$mountpoint" "$overlay" + fi +done + +echo "Done!" From b7c29e6648439a97f674c6ddd71706745e2d06cd Mon Sep 17 00:00:00 2001 From: gw0 Date: Tue, 18 Nov 2025 20:26:39 +0100 Subject: [PATCH 04/22] badgerdb-sampler: Change JSON output parameters --- badgerdb-sampler/main.go | 25 ++++---- .../scripts/run-mainnet-samplers.sh | 56 ++++++++--------- .../scripts/run-testnet-samplers.sh | 60 +++++++++---------- 3 files changed, 70 insertions(+), 71 deletions(-) diff --git a/badgerdb-sampler/main.go b/badgerdb-sampler/main.go index 8033a0d..9649c3b 100644 --- a/badgerdb-sampler/main.go +++ b/badgerdb-sampler/main.go @@ -44,20 +44,20 @@ type DBStats struct { func main() { if len(os.Args) < 3 { - fmt.Fprintf(os.Stderr, "Usage: %s [output-prefix] [max-samples]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s [output-json] [max-samples]\n", os.Args[0]) fmt.Fprintf(os.Stderr, "Database types: consensus-blockstore, consensus-evidence, consensus-mkvs, consensus-state, runtime-mkvs, runtime-history\n") fmt.Fprintf(os.Stderr, "Example: %s consensus-blockstore /path/to/blockstore.badger.db\n", os.Args[0]) - fmt.Fprintf(os.Stderr, "Example: %s runtime-mkvs /path/to/mkvs_storage.badger.db ./outputs/testnet-20220303/emerald- 500\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Example: %s runtime-mkvs /path/to/mkvs_storage.badger.db ./outputs/testnet-20220303/emerald-runtime-mkvs.json 500\n", os.Args[0]) os.Exit(1) } // Parse arguments dbType := normalizeDBType(os.Args[1]) dbPath := os.Args[2] - outputPrefix := "" + jsonFile := "" maxSamples := 200 // default if len(os.Args) >= 4 { - outputPrefix = os.Args[3] + jsonFile = os.Args[3] } if len(os.Args) >= 5 { var err error @@ -103,19 +103,18 @@ func main() { fmt.Println(string(output)) - // Save results in JSON files in per-snapshot dir and per-database type with optional prefix - if outputPrefix != "" { - outputDir := filepath.Dir(outputPrefix) - err = os.MkdirAll(outputDir, 0755) + // Export to JSON if requested + if jsonFile != "" { + jsonDir := filepath.Dir(jsonFile) + err = os.MkdirAll(jsonDir, 0755) if err != nil { - log.Printf("Warning: Could not create directory %s: %v", outputDir, err) + log.Printf("Warning: Failed to create directory %s: %v", jsonDir, err) } else { - outputFile := outputPrefix + dbType + ".json" - err = os.WriteFile(outputFile, output, 0644) + err = os.WriteFile(jsonFile, output, 0644) if err != nil { - log.Printf("Warning: Could not write to %s: %v", outputFile, err) + fmt.Fprintf(os.Stderr, "\nWarning: Failed to write JSON file %s: %v\n", jsonFile, err) } else { - fmt.Fprintf(os.Stderr, "\nResults saved to: %s\n", outputFile) + fmt.Fprintf(os.Stderr, "\n✓ Results exported to: %s\n", jsonFile) } } } diff --git a/badgerdb-sampler/scripts/run-mainnet-samplers.sh b/badgerdb-sampler/scripts/run-mainnet-samplers.sh index 5ed2b19..86154f3 100755 --- a/badgerdb-sampler/scripts/run-mainnet-samplers.sh +++ b/badgerdb-sampler/scripts/run-mainnet-samplers.sh @@ -11,51 +11,51 @@ mkdir -p "$OUTPUT" # $OUTPUT/mainnet/20201001-20201118/ (BadgerDB v2) ( mkdir -p "$OUTPUT/mainnet-20201001-20201118" -$SAMPLER_V2 consensus-blockstore-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20201001-20201118/ > $OUTPUT/mainnet-20201001-20201118/consensus-blockstore-v2.log 2>&1 || true -$SAMPLER_V2 consensus-evidence-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20201001-20201118/ > $OUTPUT/mainnet-20201001-20201118/consensus-evidence-v2.log 2>&1 || true -$SAMPLER_V2 consensus-state-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20201001-20201118/ > $OUTPUT/mainnet-20201001-20201118/consensus-state-v2.log 2>&1 || true -$SAMPLER_V2 consensus-mkvs-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20201001-20201118/ > $OUTPUT/mainnet-20201001-20201118/consensus-mkvs-v2.log 2>&1 || true +$SAMPLER_V2 consensus-blockstore-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20201001-20201118/consensus-blockstore-v2.json > $OUTPUT/mainnet-20201001-20201118/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20201001-20201118/consensus-evidence-v2.json > $OUTPUT/mainnet-20201001-20201118/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20201001-20201118/consensus-state-v2.json > $OUTPUT/mainnet-20201001-20201118/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20201001-20201118/consensus-mkvs-v2.json > $OUTPUT/mainnet-20201001-20201118/consensus-mkvs-v2.log 2>&1 || true ) & # $OUTPUT/mainnet/20201118-20210428/ (BadgerDB v2) ( mkdir -p "$OUTPUT/mainnet-20201118-20210428" -$SAMPLER_V2 consensus-blockstore-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20201118-20210428/ > $OUTPUT/mainnet-20201118-20210428/consensus-blockstore-v2.log 2>&1 || true -$SAMPLER_V2 consensus-evidence-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20201118-20210428/ > $OUTPUT/mainnet-20201118-20210428/consensus-evidence-v2.log 2>&1 || true -$SAMPLER_V2 consensus-state-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20201118-20210428/ > $OUTPUT/mainnet-20201118-20210428/consensus-state-v2.log 2>&1 || true -$SAMPLER_V2 consensus-mkvs-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20201118-20210428/ > $OUTPUT/mainnet-20201118-20210428/consensus-mkvs-v2.log 2>&1 || true +$SAMPLER_V2 consensus-blockstore-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20201118-20210428/consensus-blockstore-v2.json > $OUTPUT/mainnet-20201118-20210428/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20201118-20210428/consensus-evidence-v2.json > $OUTPUT/mainnet-20201118-20210428/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20201118-20210428/consensus-state-v2.json > $OUTPUT/mainnet-20201118-20210428/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20201118-20210428/consensus-mkvs-v2.json > $OUTPUT/mainnet-20201118-20210428/consensus-mkvs-v2.log 2>&1 || true ) & # $OUTPUT/mainnet/20210428-20220411/ (BadgerDB v3) ( mkdir -p "$OUTPUT/mainnet-20210428-20220411" -$SAMPLER_V3 consensus-blockstore-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20210428-20220411/ > $OUTPUT/mainnet-20210428-20220411/consensus-blockstore-v3.log 2>&1 || true -$SAMPLER_V3 consensus-evidence-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20210428-20220411/ > $OUTPUT/mainnet-20210428-20220411/consensus-evidence-v3.log 2>&1 || true -$SAMPLER_V3 consensus-state-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20210428-20220411/ > $OUTPUT/mainnet-20210428-20220411/consensus-state-v3.log 2>&1 || true -$SAMPLER_V3 consensus-mkvs-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/ > $OUTPUT/mainnet-20210428-20220411/consensus-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 consensus-blockstore-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20210428-20220411/consensus-blockstore-v3.json > $OUTPUT/mainnet-20210428-20220411/consensus-blockstore-v3.log 2>&1 || true +$SAMPLER_V3 consensus-evidence-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20210428-20220411/consensus-evidence-v3.json > $OUTPUT/mainnet-20210428-20220411/consensus-evidence-v3.log 2>&1 || true +$SAMPLER_V3 consensus-state-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20210428-20220411/consensus-state-v3.json > $OUTPUT/mainnet-20210428-20220411/consensus-state-v3.log 2>&1 || true +$SAMPLER_V3 consensus-mkvs-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/consensus-mkvs-v3.json > $OUTPUT/mainnet-20210428-20220411/consensus-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20210428-20220411/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/ cipher > $OUTPUT/mainnet-20210428-20220411/cipher-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20210428-20220411/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/ emerald > $OUTPUT/mainnet-20210428-20220411/emerald-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20210428-20220411/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/cipher-runtime-mkvs-v3.json > $OUTPUT/mainnet-20210428-20220411/cipher-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20210428-20220411/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/emerald-runtime-mkvs-v3.json > $OUTPUT/mainnet-20210428-20220411/emerald-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20210428-20220411/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT/mainnet-20210428-20220411/ cipher > $OUTPUT/mainnet-20210428-20220411/cipher-runtime-history-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20210428-20220411/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT/mainnet-20210428-20220411/ emerald > $OUTPUT/mainnet-20210428-20220411/emerald-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20210428-20220411/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT/mainnet-20210428-20220411/cipher-runtime-history-v3.json > $OUTPUT/mainnet-20210428-20220411/cipher-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20210428-20220411/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT/mainnet-20210428-20220411/emerald-runtime-history-v3.json > $OUTPUT/mainnet-20210428-20220411/emerald-runtime-history-v3.log 2>&1 || true ) & # $OUTPUT/mainnet/20220411-20231129/ (BadgerDB v3) ( mkdir -p "$OUTPUT/mainnet-20220411-20231129" -$SAMPLER_V3 consensus-blockstore-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20220411-20231129/ > $OUTPUT/mainnet-20220411-20231129/consensus-blockstore-v3.log 2>&1 || true -$SAMPLER_V3 consensus-evidence-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20220411-20231129/ > $OUTPUT/mainnet-20220411-20231129/consensus-evidence-v3.log 2>&1 || true -$SAMPLER_V3 consensus-state-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20220411-20231129/ > $OUTPUT/mainnet-20220411-20231129/consensus-state-v3.log 2>&1 || true -$SAMPLER_V3 consensus-mkvs-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/ > $OUTPUT/mainnet-20220411-20231129/consensus-mkvs-v3.log 2>&1 || true - -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/cipher- > $OUTPUT/mainnet-20220411-20231129/cipher-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/emerald- > $OUTPUT/mainnet-20220411-20231129/emerald-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/sapphire.mount/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/sapphire- > $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-mkvs-v3.log 2>&1 || true - -$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT/mainnet-20220411-20231129/cipher- > $OUTPUT/mainnet-20220411-20231129/cipher-runtime-history-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT/mainnet-20220411-20231129/emerald- > $OUTPUT/mainnet-20220411-20231129/emerald-runtime-history-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/sapphire.mount/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/history.db $OUTPUT/mainnet-20220411-20231129/sapphire- > $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 consensus-blockstore-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20220411-20231129/consensus-blockstore-v3.json > $OUTPUT/mainnet-20220411-20231129/consensus-blockstore-v3.log 2>&1 || true +$SAMPLER_V3 consensus-evidence-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20220411-20231129/consensus-evidence-v3.json > $OUTPUT/mainnet-20220411-20231129/consensus-evidence-v3.log 2>&1 || true +$SAMPLER_V3 consensus-state-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20220411-20231129/consensus-state-v3.json > $OUTPUT/mainnet-20220411-20231129/consensus-state-v3.log 2>&1 || true +$SAMPLER_V3 consensus-mkvs-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/consensus-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/consensus-mkvs-v3.log 2>&1 || true + +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/cipher-runtime-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/cipher-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/emerald-runtime-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/emerald-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/sapphire.mount/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-mkvs-v3.log 2>&1 || true + +$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT/mainnet-20220411-20231129/cipher-runtime-history-v3.json > $OUTPUT/mainnet-20220411-20231129/cipher-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT/mainnet-20220411-20231129/emerald-runtime-history-v3.json > $OUTPUT/mainnet-20220411-20231129/emerald-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/sapphire.mount/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/history.db $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-history-v3.json > $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-history-v3.log 2>&1 || true ) & echo "All samplers launched in background!" diff --git a/badgerdb-sampler/scripts/run-testnet-samplers.sh b/badgerdb-sampler/scripts/run-testnet-samplers.sh index 79f1811..34573f3 100755 --- a/badgerdb-sampler/scripts/run-testnet-samplers.sh +++ b/badgerdb-sampler/scripts/run-testnet-samplers.sh @@ -11,63 +11,63 @@ mkdir -p "$OUTPUT" # $OUTPUT/testnet/20200915-20201104/ (BadgerDB v2) ( mkdir -p "$OUTPUT/testnet-20200915-20201104" -$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20200915-20201104/ > $OUTPUT/testnet-20200915-20201104/consensus-blockstore-v2.log 2>&1 || true -$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20200915-20201104/ > $OUTPUT/testnet-20200915-20201104/consensus-evidence-v2.log 2>&1 || true -$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20200915-20201104/ > $OUTPUT/testnet-20200915-20201104/consensus-state-v2.log 2>&1 || true -$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20200915-20201104/ > $OUTPUT/testnet-20200915-20201104/consensus-mkvs-v2.log 2>&1 || true +$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20200915-20201104/consensus-blockstore-v2.json > $OUTPUT/testnet-20200915-20201104/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20200915-20201104/consensus-evidence-v2.json > $OUTPUT/testnet-20200915-20201104/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20200915-20201104/consensus-state-v2.json > $OUTPUT/testnet-20200915-20201104/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20200915-20201104/consensus-mkvs-v2.json > $OUTPUT/testnet-20200915-20201104/consensus-mkvs-v2.log 2>&1 || true ) & # $OUTPUT/testnet/20201104-20210203/ (BadgerDB v2) ( mkdir -p "$OUTPUT/testnet-20201104-20210203" -$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20201104-20210203/ > $OUTPUT/testnet-20201104-20210203/consensus-blockstore-v2.log 2>&1 || true -$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20201104-20210203/ > $OUTPUT/testnet-20201104-20210203/consensus-evidence-v2.log 2>&1 || true -$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20201104-20210203/ > $OUTPUT/testnet-20201104-20210203/consensus-state-v2.log 2>&1 || true -$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20201104-20210203/ > $OUTPUT/testnet-20201104-20210203/consensus-mkvs-v2.log 2>&1 || true +$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20201104-20210203/consensus-blockstore-v2.json > $OUTPUT/testnet-20201104-20210203/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20201104-20210203/consensus-evidence-v2.json > $OUTPUT/testnet-20201104-20210203/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20201104-20210203/consensus-state-v2.json > $OUTPUT/testnet-20201104-20210203/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20201104-20210203/consensus-mkvs-v2.json > $OUTPUT/testnet-20201104-20210203/consensus-mkvs-v2.log 2>&1 || true ) & # $OUTPUT/testnet/20210203-20210324/ (BadgerDB v2) ( mkdir -p "$OUTPUT/testnet-20210203-20210324" -$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210203-20210324/ > $OUTPUT/testnet-20210203-20210324/consensus-blockstore-v2.log 2>&1 || true -$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210203-20210324/ > $OUTPUT/testnet-20210203-20210324/consensus-evidence-v2.log 2>&1 || true -$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210203-20210324/ > $OUTPUT/testnet-20210203-20210324/consensus-state-v2.log 2>&1 || true -$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20210203-20210324/ > $OUTPUT/testnet-20210203-20210324/consensus-mkvs-v2.log 2>&1 || true +$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210203-20210324/consensus-blockstore-v2.json > $OUTPUT/testnet-20210203-20210324/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210203-20210324/consensus-evidence-v2.json > $OUTPUT/testnet-20210203-20210324/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210203-20210324/consensus-state-v2.json > $OUTPUT/testnet-20210203-20210324/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20210203-20210324/consensus-mkvs-v2.json > $OUTPUT/testnet-20210203-20210324/consensus-mkvs-v2.log 2>&1 || true ) & # $OUTPUT/testnet/20210324-20210413/ (BadgerDB v2) ( mkdir -p "$OUTPUT/testnet-20210324-20210413" -$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210324-20210413/ > $OUTPUT/testnet-20210324-20210413/consensus-blockstore-v2.log 2>&1 || true -$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210324-20210413/ > $OUTPUT/testnet-20210324-20210413/consensus-evidence-v2.log 2>&1 || true -$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210324-20210413/ > $OUTPUT/testnet-20210324-20210413/consensus-state-v2.log 2>&1 || true -$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20210324-20210413/ > $OUTPUT/testnet-20210324-20210413/consensus-mkvs-v2.log 2>&1 || true +$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210324-20210413/consensus-blockstore-v2.json > $OUTPUT/testnet-20210324-20210413/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210324-20210413/consensus-evidence-v2.json > $OUTPUT/testnet-20210324-20210413/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210324-20210413/consensus-state-v2.json > $OUTPUT/testnet-20210324-20210413/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20210324-20210413/consensus-mkvs-v2.json > $OUTPUT/testnet-20210324-20210413/consensus-mkvs-v2.log 2>&1 || true ) & # $OUTPUT/testnet/20210413-20220303/ (BadgerDB v3) ( mkdir -p "$OUTPUT/testnet-20210413-20220303" -$SAMPLER_V3 consensus-blockstore-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210413-20220303/ > $OUTPUT/testnet-20210413-20220303/consensus-blockstore-v3.log 2>&1 || true -$SAMPLER_V3 consensus-evidence-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210413-20220303/ > $OUTPUT/testnet-20210413-20220303/consensus-evidence-v3.log 2>&1 || true -$SAMPLER_V3 consensus-state-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210413-20220303/ > $OUTPUT/testnet-20210413-20220303/consensus-state-v3.log 2>&1 || true -$SAMPLER_V3 consensus-mkvs-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20210413-20220303/ > $OUTPUT/testnet-20210413-20220303/consensus-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 consensus-blockstore-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210413-20220303/consensus-blockstore-v3.json > $OUTPUT/testnet-20210413-20220303/consensus-blockstore-v3.log 2>&1 || true +$SAMPLER_V3 consensus-evidence-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210413-20220303/consensus-evidence-v3.json > $OUTPUT/testnet-20210413-20220303/consensus-evidence-v3.log 2>&1 || true +$SAMPLER_V3 consensus-state-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210413-20220303/consensus-state-v3.json > $OUTPUT/testnet-20210413-20220303/consensus-state-v3.log 2>&1 || true +$SAMPLER_V3 consensus-mkvs-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20210413-20220303/consensus-mkvs-v3.json > $OUTPUT/testnet-20210413-20220303/consensus-mkvs-v3.log 2>&1 || true ) & # $OUTPUT/testnet/20220303-20231012/ (BadgerDB v3) ( mkdir -p "$OUTPUT/testnet-20220303-20231012" -$SAMPLER_V3 consensus-blockstore-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20220303-20231012/ > $OUTPUT/testnet-20220303-20231012/consensus-blockstore-v3.log 2>&1 || true -$SAMPLER_V3 consensus-evidence-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20220303-20231012/ > $OUTPUT/testnet-20220303-20231012/consensus-evidence-v3.log 2>&1 || true -$SAMPLER_V3 consensus-state-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20220303-20231012/ > $OUTPUT/testnet-20220303-20231012/consensus-state-v3.log 2>&1 || true -$SAMPLER_V3 consensus-mkvs-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/ > $OUTPUT/testnet-20220303-20231012/consensus-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 consensus-blockstore-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20220303-20231012/consensus-blockstore-v3.json > $OUTPUT/testnet-20220303-20231012/consensus-blockstore-v3.log 2>&1 || true +$SAMPLER_V3 consensus-evidence-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20220303-20231012/consensus-evidence-v3.json > $OUTPUT/testnet-20220303-20231012/consensus-evidence-v3.log 2>&1 || true +$SAMPLER_V3 consensus-state-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20220303-20231012/consensus-state-v3.json > $OUTPUT/testnet-20220303-20231012/consensus-state-v3.log 2>&1 || true +$SAMPLER_V3 consensus-mkvs-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/consensus-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/consensus-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/cipher_testnet.mount/runtimes/0000000000000000000000000000000000000000000000000000000000000000/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/cipher- > $OUTPUT/testnet-20220303-20231012/cipher-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/emerald_testnet.mount/runtimes/00000000000000000000000000000000000000000000000072c8215e60d5bca7/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/emerald- > $OUTPUT/testnet-20220303-20231012/emerald-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/sapphire_testnet.mount/runtimes/000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/sapphire- > $OUTPUT/testnet-20220303-20231012/sapphire-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/cipher_testnet.mount/runtimes/0000000000000000000000000000000000000000000000000000000000000000/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/cipher-runtime-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/cipher-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/emerald_testnet.mount/runtimes/00000000000000000000000000000000000000000000000072c8215e60d5bca7/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/emerald-runtime-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/emerald-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/sapphire_testnet.mount/runtimes/000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/sapphire-runtime-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/sapphire-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/cipher_testnet.mount/runtimes/0000000000000000000000000000000000000000000000000000000000000000/history.db $OUTPUT/testnet-20220303-20231012/cipher- > $OUTPUT/testnet-20220303-20231012/cipher-runtime-history-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/emerald_testnet.mount/runtimes/00000000000000000000000000000000000000000000000072c8215e60d5bca7/history.db $OUTPUT/testnet-20220303-20231012/emerald- > $OUTPUT/testnet-20220303-20231012/emerald-runtime-history-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/sapphire_testnet.mount/runtimes/000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c/history.db $OUTPUT/testnet-20220303-20231012/sapphire- > $OUTPUT/testnet-20220303-20231012/sapphire-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/cipher_testnet.mount/runtimes/0000000000000000000000000000000000000000000000000000000000000000/history.db $OUTPUT/testnet-20220303-20231012/cipher-runtime-history-v3.json > $OUTPUT/testnet-20220303-20231012/cipher-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/emerald_testnet.mount/runtimes/00000000000000000000000000000000000000000000000072c8215e60d5bca7/history.db $OUTPUT/testnet-20220303-20231012/emerald-runtime-history-v3.json > $OUTPUT/testnet-20220303-20231012/emerald-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/sapphire_testnet.mount/runtimes/000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c/history.db $OUTPUT/testnet-20220303-20231012/sapphire-runtime-history-v3.json > $OUTPUT/testnet-20220303-20231012/sapphire-runtime-history-v3.log 2>&1 || true ) & echo "All samplers launched in background!" From dbbea60adce881a0322b02f73b95a45c66a82608 Mon Sep 17 00:00:00 2001 From: gw0 Date: Wed, 19 Nov 2025 10:30:58 +0100 Subject: [PATCH 05/22] badgerdb-sampler: Implement local symlink mirror fallback mode and unify database code --- badgerdb-sampler/db_common.go | 97 +++++++++++++++++++++++++++++++++++ badgerdb-sampler/db_v2.go | 61 +++++++++++++--------- badgerdb-sampler/db_v3.go | 61 +++++++++++++--------- badgerdb-sampler/db_v4.go | 60 +++++++++++++--------- badgerdb-sampler/go.mod | 21 ++++++-- badgerdb-sampler/main.go | 48 +++++++++-------- 6 files changed, 249 insertions(+), 99 deletions(-) create mode 100644 badgerdb-sampler/db_common.go diff --git a/badgerdb-sampler/db_common.go b/badgerdb-sampler/db_common.go new file mode 100644 index 0000000..acc3a1d --- /dev/null +++ b/badgerdb-sampler/db_common.go @@ -0,0 +1,97 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// createLocalMirror creates a temporary local directory with database files copied/symlinked +// to avoid mmap issues on FUSE filesystems. Returns temp path and cleanup function. +func createLocalMirror(fusePath string) (string, func(), error) { + // Create temp directory on local filesystem (not FUSE) + tmpDir, err := os.MkdirTemp("", "badgerdb-temp-*") + if err != nil { + return "", nil, fmt.Errorf("failed to create temp directory: %w", err) + } + + cleanup := func() { + if err := os.RemoveAll(tmpDir); err != nil { + fmt.Fprintf(os.Stderr, "Warning: Failed to clean up temp directory %s: %v\n", tmpDir, err) + } + } + + fmt.Fprintf(os.Stderr, " Creating local mirror in: %s\n", tmpDir) + + entries, err := os.ReadDir(fusePath) + if err != nil { + cleanup() + return "", nil, fmt.Errorf("failed to read database directory: %w", err) + } + + copiedCount := 0 + linkedCount := 0 + skippedCount := 0 + + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + srcPath := filepath.Join(fusePath, name) + dstPath := filepath.Join(tmpDir, name) + + // Skip memtable files - these will be created fresh + if strings.HasSuffix(name, ".mem") { + fmt.Fprintf(os.Stderr, " Skipping memtable file: %s\n", name) + skippedCount++ + continue + } + + // Copy small metadata files (MANIFEST, DISCARD, etc.) + // Symlink large data files (*.sst, *.vlog) + if strings.HasSuffix(name, ".sst") || strings.HasSuffix(name, ".vlog") { + // Symlink large data files + if err := os.Symlink(srcPath, dstPath); err != nil { + cleanup() + return "", nil, fmt.Errorf("failed to symlink %s: %w", name, err) + } + linkedCount++ + } else { + // Copy small metadata files + if err := copyFile(srcPath, dstPath); err != nil { + cleanup() + return "", nil, fmt.Errorf("failed to copy %s: %w", name, err) + } + copiedCount++ + } + } + + fmt.Fprintf(os.Stderr, " Mirror created: %d files copied, %d files symlinked, %d files skipped\n", + copiedCount, linkedCount, skippedCount) + + return tmpDir, cleanup, nil +} + +// copyFile copies a file from src to dst +func copyFile(src, dst string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + dstFile, err := os.Create(dst) + if err != nil { + return err + } + defer dstFile.Close() + + if _, err := io.Copy(dstFile, srcFile); err != nil { + return err + } + + return dstFile.Sync() +} diff --git a/badgerdb-sampler/db_v2.go b/badgerdb-sampler/db_v2.go index 1c614c4..b005ef6 100644 --- a/badgerdb-sampler/db_v2.go +++ b/badgerdb-sampler/db_v2.go @@ -8,15 +8,14 @@ import ( "strings" badger "github.com/dgraph-io/badger/v2" - "github.com/dgraph-io/badger/v2/options" ) type ( - DB = badger.DB - Txn = badger.Txn - Item = badger.Item - Iterator = badger.Iterator - IteratorOptions = badger.IteratorOptions + DB = badger.DB + Txn = badger.Txn + Item = badger.Item + Iterator = badger.Iterator + IteratorOptions = badger.IteratorOptions ) var DefaultIteratorOptions = badger.DefaultIteratorOptions @@ -26,7 +25,7 @@ func init() { } // openDatabase tries to open the database in ReadOnly mode first, -// then falls back to FileIO mode for corrupted/improperly-closed databases +// then falls back to local mirror mode for corrupted/improperly-closed databases on FUSE func openDatabase(path string) (*DB, error) { // Try ReadOnly mode first (clean databases) opts := badger.DefaultOptions(path) @@ -35,39 +34,53 @@ func openDatabase(path string) (*DB, error) { db, err := badger.Open(opts) if err == nil { + // Successfully opened fmt.Fprintf(os.Stderr, "Opened database in ReadOnly mode\n") return db, nil } - // Check for BadgerDB v3/v4 incompatibility - if strings.Contains(err.Error(), "manifest has unsupported version") { - return nil, fmt.Errorf("database uses BadgerDB v3 or v4 (incompatible). Please use badgerdb-sampler-v3 or badgerdb-sampler-v4 instead.\nOriginal error: %v", err) + // Check for version incompatibility + if strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) } - fmt.Fprintf(os.Stderr, "ReadOnly mode failed, trying FileIO workaround...\n") + fmt.Fprintf(os.Stderr, "Failed, trying local mirror workaround for FUSE...\n") - // FileIO workaround mode - for improperly closed databases - opts = badger.DefaultOptions(path) + // Create local mirror with symlinks to avoid mmap SIGBUS errors on FUSE + tmpDir, cleanup, err := createLocalMirror(path) + if err != nil { + return nil, fmt.Errorf("failed to create local mirror: %w", err) + } + // Note: cleanup is not called here - tmp files remain in /tmp and will be cleaned by OS + // This is acceptable for read-only analysis tools. For long-running services, use defer cleanup() + _ = cleanup + + // Open from local mirror in read-write mode (needed for corrupted/improperly closed DBs) + // Unified configuration across all BadgerDB versions + opts = badger.DefaultOptions(tmpDir) opts.ReadOnly = false opts.SyncWrites = false - opts.NumMemtables = 5 + opts.NumMemtables = 1 // Minimize memtable usage + opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) + opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 opts.NumCompactors = 0 opts.CompactL0OnClose = false - opts.ValueLogLoadingMode = options.FileIO - opts.TableLoadingMode = options.FileIO + opts.BypassLockGuard = true // Allow concurrent access + opts.DetectConflicts = false // Reduce overhead opts.Logger = nil db, err = badger.Open(opts) - if err != nil { - // Check again for v3/v4 incompatibility in FileIO mode - if strings.Contains(err.Error(), "manifest has unsupported version") { - return nil, fmt.Errorf("database uses BadgerDB v3 or v4 (incompatible). Please use badgerdb-sampler-v3 or badgerdb-sampler-v4 instead.\nOriginal error: %v", err) - } - return nil, fmt.Errorf("failed to open database with both ReadOnly and FileIO modes: %v", err) + if err == nil { + // Successfully opened + fmt.Fprintf(os.Stderr, "Opened database from local mirror (tmp dir: %s)\n", tmpDir) + return db, nil } - fmt.Fprintf(os.Stderr, "Opened database in FileIO mode\n") - return db, nil + // Check again for version incompatibility + if strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) + } + return nil, fmt.Errorf("failed to open database from local mirror: %v", err) } diff --git a/badgerdb-sampler/db_v3.go b/badgerdb-sampler/db_v3.go index 8b0a2c0..9dbc84c 100644 --- a/badgerdb-sampler/db_v3.go +++ b/badgerdb-sampler/db_v3.go @@ -11,11 +11,11 @@ import ( ) type ( - DB = badger.DB - Txn = badger.Txn - Item = badger.Item - Iterator = badger.Iterator - IteratorOptions = badger.IteratorOptions + DB = badger.DB + Txn = badger.Txn + Item = badger.Item + Iterator = badger.Iterator + IteratorOptions = badger.IteratorOptions ) var DefaultIteratorOptions = badger.DefaultIteratorOptions @@ -25,7 +25,7 @@ func init() { } // openDatabase tries to open the database in ReadOnly mode first, -// then falls back to FileIO mode for corrupted/improperly-closed databases +// then falls back to local mirror mode for corrupted/improperly-closed databases on FUSE func openDatabase(path string) (*DB, error) { // Try ReadOnly mode first (clean databases) opts := badger.DefaultOptions(path) @@ -38,36 +38,49 @@ func openDatabase(path string) (*DB, error) { return db, nil } - // Check for BadgerDB v2/v4 incompatibility - if strings.Contains(err.Error(), "manifest has unsupported version") || - strings.Contains(err.Error(), "unsupported version") { - return nil, fmt.Errorf("database may use BadgerDB v2 or v4 (incompatible). Please use badgerdb-sampler-v2 or badgerdb-sampler-v4 instead.\nOriginal error: %v", err) + // Check for version incompatibility + if strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) } - fmt.Fprintf(os.Stderr, "ReadOnly mode failed, trying FileIO workaround...\n") + fmt.Fprintf(os.Stderr, "Failed, trying local mirror workaround for FUSE...\n") - // FileIO workaround mode - for improperly closed databases - opts = badger.DefaultOptions(path) + // Create local mirror with symlinks to avoid mmap SIGBUS errors on FUSE + // BadgerDB v3 always uses mmap for memtables, which doesn't work on FUSE filesystems + tmpDir, cleanup, err := createLocalMirror(path) + if err != nil { + return nil, fmt.Errorf("failed to create local mirror: %w", err) + } + // Note: cleanup is not called here - tmp files remain in /tmp and will be cleaned by OS + // This is acceptable for read-only analysis tools. For long-running services, use defer cleanup() + _ = cleanup + + // Open from local mirror in read-write mode (needed for corrupted/improperly closed DBs) + // Unified configuration across all BadgerDB versions + opts = badger.DefaultOptions(tmpDir) opts.ReadOnly = false opts.SyncWrites = false - opts.NumMemtables = 5 + opts.NumMemtables = 1 // Minimize memtable usage + opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) + opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 opts.NumCompactors = 0 opts.CompactL0OnClose = false - // Note: ValueLogLoadingMode and TableLoadingMode were removed in BadgerDB v3 + opts.BypassLockGuard = true // Allow concurrent access + opts.DetectConflicts = false // Reduce overhead opts.Logger = nil db, err = badger.Open(opts) - if err != nil { - // Check again for v2/v4 incompatibility in FileIO mode - if strings.Contains(err.Error(), "manifest has unsupported version") || - strings.Contains(err.Error(), "unsupported version") { - return nil, fmt.Errorf("database may use BadgerDB v2 or v4 (incompatible). Please use badgerdb-sampler-v2 or badgerdb-sampler-v4 instead.\nOriginal error: %v", err) - } - return nil, fmt.Errorf("failed to open database with both ReadOnly and FileIO modes: %v", err) + if err == nil { + // Successfully opened + fmt.Fprintf(os.Stderr, "Opened database from local mirror (tmp dir: %s)\n", tmpDir) + return db, nil } - fmt.Fprintf(os.Stderr, "Opened database in FileIO mode\n") - return db, nil + // Check again for version incompatibility + if strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) + } + return nil, fmt.Errorf("failed to open database from local mirror: %v", err) } diff --git a/badgerdb-sampler/db_v4.go b/badgerdb-sampler/db_v4.go index 27a35e5..3f8008c 100644 --- a/badgerdb-sampler/db_v4.go +++ b/badgerdb-sampler/db_v4.go @@ -11,11 +11,11 @@ import ( ) type ( - DB = badger.DB - Txn = badger.Txn - Item = badger.Item - Iterator = badger.Iterator - IteratorOptions = badger.IteratorOptions + DB = badger.DB + Txn = badger.Txn + Item = badger.Item + Iterator = badger.Iterator + IteratorOptions = badger.IteratorOptions ) var DefaultIteratorOptions = badger.DefaultIteratorOptions @@ -25,7 +25,7 @@ func init() { } // openDatabase tries to open the database in ReadOnly mode first, -// then falls back to FileIO mode for corrupted/improperly-closed databases +// then falls back to local mirror mode for corrupted/improperly-closed databases on FUSE func openDatabase(path string) (*DB, error) { // Try ReadOnly mode first (clean databases) opts := badger.DefaultOptions(path) @@ -38,35 +38,49 @@ func openDatabase(path string) (*DB, error) { return db, nil } - // Check for BadgerDB v2/v3 incompatibility - if strings.Contains(err.Error(), "manifest has unsupported version") || - strings.Contains(err.Error(), "unsupported version") { - return nil, fmt.Errorf("database may use BadgerDB v2 or v3 (incompatible). Please use badgerdb-sampler-v2 or badgerdb-sampler-v3 instead.\nOriginal error: %v", err) + // Check for version incompatibility + if strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) } - fmt.Fprintf(os.Stderr, "ReadOnly mode failed, trying FileIO workaround...\n") + fmt.Fprintf(os.Stderr, "Failed, trying local mirror workaround for FUSE...\n") - // FileIO workaround mode - for improperly closed databases - opts = badger.DefaultOptions(path) + // Create local mirror with symlinks to avoid mmap SIGBUS errors on FUSE + // BadgerDB v4 uses mmap for memtables similar to v3, which doesn't work on FUSE filesystems + tmpDir, cleanup, err := createLocalMirror(path) + if err != nil { + return nil, fmt.Errorf("failed to create local mirror: %w", err) + } + // Note: cleanup is not called here - tmp files remain in /tmp and will be cleaned by OS + // This is acceptable for read-only analysis tools. For long-running services, use defer cleanup() + _ = cleanup + + // Open from local mirror in read-write mode (needed for corrupted/improperly closed DBs) + // Unified configuration across all BadgerDB versions + opts = badger.DefaultOptions(tmpDir) opts.ReadOnly = false opts.SyncWrites = false - opts.NumMemtables = 5 + opts.NumMemtables = 1 // Minimize memtable usage + opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) + opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 opts.NumCompactors = 0 opts.CompactL0OnClose = false + opts.BypassLockGuard = true // Allow concurrent access + opts.DetectConflicts = false // Reduce overhead opts.Logger = nil db, err = badger.Open(opts) - if err != nil { - // Check again for v2/v3 incompatibility in FileIO mode - if strings.Contains(err.Error(), "manifest has unsupported version") || - strings.Contains(err.Error(), "unsupported version") { - return nil, fmt.Errorf("database may use BadgerDB v2 or v3 (incompatible). Please use badgerdb-sampler-v2 or badgerdb-sampler-v3 instead.\nOriginal error: %v", err) - } - return nil, fmt.Errorf("failed to open database with both ReadOnly and FileIO modes: %v", err) + if err == nil { + // Successfully opened + fmt.Fprintf(os.Stderr, "Opened database from local mirror (tmp dir: %s)\n", tmpDir) + return db, nil } - fmt.Fprintf(os.Stderr, "Opened database in FileIO mode\n") - return db, nil + // Check again for version incompatibility + if strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) + } + return nil, fmt.Errorf("failed to open database from local mirror: %v", err) } diff --git a/badgerdb-sampler/go.mod b/badgerdb-sampler/go.mod index f9332c6..c3044af 100644 --- a/badgerdb-sampler/go.mod +++ b/badgerdb-sampler/go.mod @@ -3,10 +3,25 @@ module github.com/oasisprotocol/badgerdb-sampler go 1.25.4 require ( - github.com/dgraph-io/badger/v4 v4.2.0 + github.com/dgraph-io/badger/v3 v3.2103.5 github.com/gogo/protobuf v1.3.2 github.com/tendermint/tendermint v0.34.21 ) -// Note: Dependencies will be filled in when v4 support is implemented -// Run 'go mod tidy' with -modfile=go.mod.v4 to populate +require ( + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/flatbuffers v1.12.1 // indirect + github.com/klauspost/compress v1.15.9 // indirect + github.com/pkg/errors v0.9.1 // indirect + go.opencensus.io v0.23.0 // indirect + golang.org/x/net v0.0.0-20220726230323-06994584191e // indirect + golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect + google.golang.org/protobuf v1.28.0 // indirect +) diff --git a/badgerdb-sampler/main.go b/badgerdb-sampler/main.go index 9649c3b..ae6a2b4 100644 --- a/badgerdb-sampler/main.go +++ b/badgerdb-sampler/main.go @@ -74,6 +74,7 @@ func main() { } // Open database with two-mode strategy (corruption handling) + fmt.Printf("Opening database %s (type: %s, size: %d bytes)...\n", dbPath, dbType, dbSize) db, err := openDatabase(dbPath) if err != nil { log.Fatalf("Failed to open database: %v", err) @@ -90,17 +91,18 @@ func main() { } // Collect samples + fmt.Printf("Collecting and decoding samples...\n") err = collectSamples(db, &stats, maxSamples) if err != nil { log.Fatalf("Error collecting samples: %v", err) } - // Output JSON + // Print results + fmt.Printf("Results:\n") output, err := json.MarshalIndent(stats, "", " ") if err != nil { log.Fatalf("Error marshaling JSON: %v", err) } - fmt.Println(string(output)) // Export to JSON if requested @@ -108,7 +110,7 @@ func main() { jsonDir := filepath.Dir(jsonFile) err = os.MkdirAll(jsonDir, 0755) if err != nil { - log.Printf("Warning: Failed to create directory %s: %v", jsonDir, err) + log.Fprintf(os.Stderr, "\nWarning: Failed to create directory %s: %v", jsonDir, err) } else { err = os.WriteFile(jsonFile, output, 0644) if err != nil { @@ -286,7 +288,6 @@ func decodeValueConsensusBlockstore(keyType string, value []byte) (string, int64 if err := proto.Unmarshal(value, &state); err == nil { return fmt.Sprintf("BlockStoreState{base: %d, height: %d}", state.Base, state.Height), 0 } - return fmt.Sprintf("Failed to decode BlockStoreState (size: %d bytes)", len(value)), 0 case "block_meta": var meta tmproto.BlockMeta @@ -315,7 +316,6 @@ func decodeValueConsensusBlockstore(keyType string, value []byte) (string, int64 return fmt.Sprintf("BlockMeta{height: %d, time: %s, chain: %s, num_txs: %d, app_hash: %s}", meta.Header.Height, ts, chainID, meta.NumTxs, appHash), tsUnix } - return fmt.Sprintf("Failed to decode BlockMeta (size: %d bytes)", len(value)), 0 case "block_part": var part tmproto.Part @@ -323,7 +323,6 @@ func decodeValueConsensusBlockstore(keyType string, value []byte) (string, int64 return fmt.Sprintf("Part{index: %d, bytes_size: %d, proof_total: %d}", part.Index, len(part.Bytes), part.Proof.Total), 0 } - return fmt.Sprintf("Failed to decode Part (size: %d bytes)", len(value)), 0 case "block_commit": var commit tmproto.Commit @@ -331,27 +330,26 @@ func decodeValueConsensusBlockstore(keyType string, value []byte) (string, int64 return fmt.Sprintf("Commit{height: %d, round: %d, signatures: %d}", commit.Height, commit.Round, len(commit.Signatures)), 0 } - return fmt.Sprintf("Failed to decode Commit (size: %d bytes)", len(value)), 0 case "seen_commit": var commit tmproto.Commit if err := proto.Unmarshal(value, &commit); err == nil { - return fmt.Sprintf("SeenCommit{height: %d, round: %d, signatures: %d}", + return fmt.Sprintf("Commit{height: %d, round: %d, signatures: %d}", commit.Height, commit.Round, len(commit.Signatures)), 0 } - return fmt.Sprintf("Failed to decode SeenCommit (size: %d bytes)", len(value)), 0 case "block_hash": // Block hash values are plain strings containing height numbers height, err := strconv.ParseInt(string(value), 10, 64) if err == nil { - return fmt.Sprintf("Height: %d", height), 0 + return fmt.Sprintf("Hash{height: %d}", height), 0 } - return fmt.Sprintf("String: %s", string(value)), 0 default: return fmt.Sprintf("Unknown type (size: %d bytes)", len(value)), 0 } + + return fmt.Sprintf("Failed to decode blockstore (type: %s, size: %d bytes)", keyType, len(value)), 0 } // analyzeKeyConsensusEvidence parses consensus-evidence key and returns key type and decoded representation @@ -382,7 +380,7 @@ func decodeValueConsensusEvidence(keyType string, value []byte) string { evidence.VoteA.Height, evidence.VoteB.Height) } - return fmt.Sprintf("Evidence (size: %d bytes, type: %s)", len(value), keyType) + return fmt.Sprintf("Evidence (type: %s, size: %d bytes)", keyType, len(value)) } // analyzeKeyConsensusMkvs parses consensus-mkvs key and returns key type and decoded representation @@ -414,7 +412,7 @@ func analyzeKeyConsensusMkvs(key []byte) (string, string) { switch prefixByte { case 0x00: // nodeKeyFmt: hash.Hash (32 bytes) - return "node", fmt.Sprintf("node:hash=%x", truncateBytes(data, 8)) + return "node", fmt.Sprintf("node{hash: %x....}", truncateBytes(data, 8)) case 0x01: // writeLogKeyFmt: uint64(version) + TypedHash(new root) + TypedHash(old root) if len(data) >= 8 { @@ -422,18 +420,18 @@ func analyzeKeyConsensusMkvs(key []byte) (string, string) { if len(data) >= 8+33 { // version + first TypedHash (33 bytes in v3+) rootType := data[8] rootHash := data[9:9+32] - return "write_log", fmt.Sprintf("write_log:v=%d,new_root_type=%d,hash=%x...", version, rootType, truncateBytes(rootHash, 4)) + return "write_log", fmt.Sprintf("write_log{v:%d, new_root_type:%d, hash: %x...}", version, rootType, truncateBytes(rootHash, 4)) } - return "write_log", fmt.Sprintf("write_log:v=%d", version) + return "write_log", fmt.Sprintf("write_log{v:%d}", version) } - return "write_log", "write_log:" + return "write_log", "write_log{}" case 0x02: // rootsMetadataKeyFmt: uint64(version) if len(data) >= 8 { version := binary.BigEndian.Uint64(data[0:8]) - return "roots_metadata", fmt.Sprintf("roots_metadata:v=%d", version) + return "roots_metadata", fmt.Sprintf("roots_metadata{v:%d}", version) } - return "roots_metadata", "roots_metadata:" + return "roots_metadata", "roots_metadata{}" case 0x03: // rootUpdatedNodesKeyFmt: uint64(version) + TypedHash(root) if len(data) >= 8 { @@ -441,11 +439,11 @@ func analyzeKeyConsensusMkvs(key []byte) (string, string) { if len(data) >= 8+33 { // version + TypedHash rootType := data[8] rootHash := data[9:9+32] - return "root_updated_nodes", fmt.Sprintf("root_updated_nodes:v=%d,root_type=%d,hash=%x...", version, rootType, truncateBytes(rootHash, 4)) + return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{v:%d, root_type:%d, hash:%x...}", version, rootType, truncateBytes(rootHash, 4)) } - return "root_updated_nodes", fmt.Sprintf("root_updated_nodes:v=%d", version) + return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{v:%d}", version) } - return "root_updated_nodes", "root_updated_nodes:" + return "root_updated_nodes", "root_updated_nodes{}" case 0x04: // metadataKeyFmt: no additional data return "metadata", "metadata" @@ -454,17 +452,17 @@ func analyzeKeyConsensusMkvs(key []byte) (string, string) { if len(data) >= 33 { rootType := data[0] rootHash := data[1:33] - return "multipart_restore_log", fmt.Sprintf("multipart_restore_log:type=%d,hash=%x", rootType, truncateBytes(rootHash, 8)) + return "multipart_restore_log", fmt.Sprintf("multipart_restore_log{type:%d, hash:%x}", rootType, truncateBytes(rootHash, 8)) } - return "multipart_restore_log", fmt.Sprintf("multipart_restore_log:hash=%x", truncateBytes(data, 8)) + return "multipart_restore_log", fmt.Sprintf("multipart_restore_log{hash:%x}", truncateBytes(data, 8)) case 0x06: // rootNodeKeyFmt: TypedHash (33 bytes) if len(data) >= 33 { rootType := data[0] rootHash := data[1:33] - return "root_node", fmt.Sprintf("root_node:type=%d,hash=%x", rootType, truncateBytes(rootHash, 8)) + return "root_node", fmt.Sprintf("root_node{type:%d, hash:%x}", rootType, truncateBytes(rootHash, 8)) } - return "root_node", fmt.Sprintf("root_node:hash=%x", truncateBytes(data, 8)) + return "root_node", fmt.Sprintf("root_node{hash:%x}", truncateBytes(data, 8)) default: return fmt.Sprintf("unknown_%02x", prefixByte), fmt.Sprintf("unknown_%02x:%x", prefixByte, truncateBytes(data, 8)) } From dc4d7ed5b47014f29756f4d4c2b629f645f116bb Mon Sep 17 00:00:00 2001 From: gw0 Date: Sat, 22 Nov 2025 06:57:31 +0100 Subject: [PATCH 06/22] badgerdb-sampler: Improve logging --- badgerdb-sampler/db_v2.go | 4 ++-- badgerdb-sampler/db_v3.go | 4 ++-- badgerdb-sampler/db_v4.go | 4 ++-- badgerdb-sampler/main.go | 16 ++++++++++++++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/badgerdb-sampler/db_v2.go b/badgerdb-sampler/db_v2.go index b005ef6..5664386 100644 --- a/badgerdb-sampler/db_v2.go +++ b/badgerdb-sampler/db_v2.go @@ -30,7 +30,7 @@ func openDatabase(path string) (*DB, error) { // Try ReadOnly mode first (clean databases) opts := badger.DefaultOptions(path) opts.ReadOnly = true - opts.Logger = nil + // Keep default logger enabled for diagnostics db, err := badger.Open(opts) if err == nil { @@ -69,7 +69,7 @@ func openDatabase(path string) (*DB, error) { opts.CompactL0OnClose = false opts.BypassLockGuard = true // Allow concurrent access opts.DetectConflicts = false // Reduce overhead - opts.Logger = nil + // Keep default logger enabled for diagnostics db, err = badger.Open(opts) if err == nil { diff --git a/badgerdb-sampler/db_v3.go b/badgerdb-sampler/db_v3.go index 9dbc84c..31c9b17 100644 --- a/badgerdb-sampler/db_v3.go +++ b/badgerdb-sampler/db_v3.go @@ -30,7 +30,7 @@ func openDatabase(path string) (*DB, error) { // Try ReadOnly mode first (clean databases) opts := badger.DefaultOptions(path) opts.ReadOnly = true - opts.Logger = nil + // Keep default logger enabled for diagnostics db, err := badger.Open(opts) if err == nil { @@ -69,7 +69,7 @@ func openDatabase(path string) (*DB, error) { opts.CompactL0OnClose = false opts.BypassLockGuard = true // Allow concurrent access opts.DetectConflicts = false // Reduce overhead - opts.Logger = nil + // Keep default logger enabled for diagnostics db, err = badger.Open(opts) if err == nil { diff --git a/badgerdb-sampler/db_v4.go b/badgerdb-sampler/db_v4.go index 3f8008c..6d80818 100644 --- a/badgerdb-sampler/db_v4.go +++ b/badgerdb-sampler/db_v4.go @@ -30,7 +30,7 @@ func openDatabase(path string) (*DB, error) { // Try ReadOnly mode first (clean databases) opts := badger.DefaultOptions(path) opts.ReadOnly = true - opts.Logger = nil + // Keep default logger enabled for diagnostics db, err := badger.Open(opts) if err == nil { @@ -69,7 +69,7 @@ func openDatabase(path string) (*DB, error) { opts.CompactL0OnClose = false opts.BypassLockGuard = true // Allow concurrent access opts.DetectConflicts = false // Reduce overhead - opts.Logger = nil + // Keep default logger enabled for diagnostics db, err = badger.Open(opts) if err == nil { diff --git a/badgerdb-sampler/main.go b/badgerdb-sampler/main.go index ae6a2b4..e0d1ef5 100644 --- a/badgerdb-sampler/main.go +++ b/badgerdb-sampler/main.go @@ -55,7 +55,7 @@ func main() { dbType := normalizeDBType(os.Args[1]) dbPath := os.Args[2] jsonFile := "" - maxSamples := 200 // default + maxSamples := 1000 // default if len(os.Args) >= 4 { jsonFile = os.Args[3] } @@ -153,16 +153,26 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { return db.View(func(txn *Txn) error { opts := DefaultIteratorOptions opts.PrefetchValues = true - opts.PrefetchSize = maxSamples + opts.PrefetchSize = 100 // Optimal prefetch size for BadgerDB + opts.AllVersions = false // Only latest version of each key it := txn.NewIterator(opts) defer it.Close() sampleCount := 0 + startTime := time.Now() + + fmt.Fprintf(os.Stderr, "Starting iteration...\n") for it.Rewind(); it.Valid(); it.Next() { if sampleCount >= maxSamples { break } + + // Progress logging every 10 samples + if sampleCount > 0 && sampleCount%10 == 0 { + elapsed := time.Since(startTime) + fmt.Fprintf(os.Stderr, " Progress: %d/%d samples collected (elapsed: %v)\n", sampleCount, maxSamples, elapsed.Round(time.Millisecond)) + } item := it.Item() key := item.Key() @@ -236,6 +246,8 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { } stats.SampleCount = sampleCount + elapsed := time.Since(startTime) + fmt.Fprintf(os.Stderr, "Iteration complete: %d samples in %v\n", sampleCount, elapsed.Round(time.Millisecond)) return nil }) } From e3cfdab70b880701762bd0e9fb669da04f9b50f2 Mon Sep 17 00:00:00 2001 From: gw0 Date: Sat, 22 Nov 2025 11:02:31 +0100 Subject: [PATCH 07/22] badgerdb-sampler: Add decoding of runtime keys and CBOR values --- badgerdb-sampler/go.mod | 2 + badgerdb-sampler/go.sum | 4 + badgerdb-sampler/main.go | 554 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 539 insertions(+), 21 deletions(-) diff --git a/badgerdb-sampler/go.mod b/badgerdb-sampler/go.mod index c3044af..1db1dea 100644 --- a/badgerdb-sampler/go.mod +++ b/badgerdb-sampler/go.mod @@ -13,6 +13,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -20,6 +21,7 @@ require ( github.com/google/flatbuffers v1.12.1 // indirect github.com/klauspost/compress v1.15.9 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/x448/float16 v0.8.4 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/net v0.0.0-20220726230323-06994584191e // indirect golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect diff --git a/badgerdb-sampler/go.sum b/badgerdb-sampler/go.sum index 4ffd7ff..89410b4 100644 --- a/badgerdb-sampler/go.sum +++ b/badgerdb-sampler/go.sum @@ -31,6 +31,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -106,6 +108,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/tendermint/tendermint v0.34.21 h1:UiGGnBFHVrZhoQVQ7EfwSOLuCtarqCSsRf8VrklqB7s= github.com/tendermint/tendermint v0.34.21/go.mod h1:XDvfg6U7grcFTDx7VkzxnhazQ/bspGJAn4DZ6DcLLjQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/badgerdb-sampler/main.go b/badgerdb-sampler/main.go index e0d1ef5..8b2de5b 100644 --- a/badgerdb-sampler/main.go +++ b/badgerdb-sampler/main.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/fxamacker/cbor/v2" "github.com/gogo/protobuf/proto" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" tmstore "github.com/tendermint/tendermint/proto/tendermint/store" @@ -42,6 +43,56 @@ type DBStats struct { Samples []Sample `json:"samples"` } +// Runtime history CBOR types (from oasis-core/go/runtime/history/db.go) + +// RuntimeHistoryMetadata represents the metadata stored in runtime history DB +type RuntimeHistoryMetadata struct { + RuntimeID []byte `cbor:"runtime_id"` + Version uint64 `cbor:"version"` + LastConsensusHeight int64 `cbor:"last_consensus_height"` + LastRound uint64 `cbor:"last_round"` +} + +// RuntimeHistoryAnnotatedBlock represents an annotated block in runtime history +type RuntimeHistoryAnnotatedBlock struct { + Height int64 `cbor:"consensus_height"` + Block *RuntimeHistoryBlock `cbor:"block"` +} + +// RuntimeHistoryBlock represents a runtime block +type RuntimeHistoryBlock struct { + Header RuntimeHistoryBlockHeader `cbor:"header"` +} + +// RuntimeHistoryBlockHeader represents a runtime block header +type RuntimeHistoryBlockHeader struct { + Version uint16 `cbor:"version"` + Namespace []byte `cbor:"namespace"` + Round uint64 `cbor:"round"` + Timestamp uint64 `cbor:"timestamp"` + HeaderType uint8 `cbor:"header_type"` + PreviousHash []byte `cbor:"previous_hash"` + IORoot []byte `cbor:"io_root"` + StateRoot []byte `cbor:"state_root"` + MessagesHash []byte `cbor:"messages_hash"` + InMessagesHash []byte `cbor:"in_msgs_hash"` +} + +// RuntimeHistoryRoundResults represents round results in runtime history +type RuntimeHistoryRoundResults struct { + Messages []RuntimeHistoryMessageEvent `cbor:"messages,omitempty"` + GoodComputeEntities [][]byte `cbor:"good_compute_entities,omitempty"` + BadComputeEntities [][]byte `cbor:"bad_compute_entities,omitempty"` +} + +// RuntimeHistoryMessageEvent represents a message event +type RuntimeHistoryMessageEvent struct { + Module string `cbor:"module,omitempty"` + Code uint32 `cbor:"code,omitempty"` + Index uint32 `cbor:"index,omitempty"` + Result cbor.RawMessage `cbor:"result,omitempty"` +} + func main() { if len(os.Args) < 3 { fmt.Fprintf(os.Stderr, "Usage: %s [output-json] [max-samples]\n", os.Args[0]) @@ -56,14 +107,14 @@ func main() { dbPath := os.Args[2] jsonFile := "" maxSamples := 1000 // default - if len(os.Args) >= 4 { + if len(os.Args) > 3 { jsonFile = os.Args[3] } - if len(os.Args) >= 5 { + if len(os.Args) > 4 { var err error maxSamples, err = strconv.Atoi(os.Args[4]) if err != nil || maxSamples <= 0 { - log.Fatalf("Invalid max-samples value: %s (must be positive integer)", os.Args[5]) + log.Fatalf("Invalid max-samples value: %s (must be positive integer)", os.Args[4]) } } @@ -110,7 +161,7 @@ func main() { jsonDir := filepath.Dir(jsonFile) err = os.MkdirAll(jsonDir, 0755) if err != nil { - log.Fprintf(os.Stderr, "\nWarning: Failed to create directory %s: %v", jsonDir, err) + fmt.Fprintf(os.Stderr, "\nWarning: Failed to create directory %s: %v\n", jsonDir, err) } else { err = os.WriteFile(jsonFile, output, 0644) if err != nil { @@ -168,8 +219,8 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { break } - // Progress logging every 10 samples - if sampleCount > 0 && sampleCount%10 == 0 { + // Progress logging every few samples + if sampleCount > 0 && sampleCount%1000 == 0 { elapsed := time.Since(startTime) fmt.Fprintf(os.Stderr, " Progress: %d/%d samples collected (elapsed: %v)\n", sampleCount, maxSamples, elapsed.Round(time.Millisecond)) } @@ -480,19 +531,197 @@ func analyzeKeyConsensusMkvs(key []byte) (string, string) { } } -// decodeValueConsensusMkvs decodes MKVS value +// decodeValueConsensusMkvs decodes consensus MKVS value with inline node parsing func decodeValueConsensusMkvs(keyType string, value []byte) string { if len(value) == 0 { return "Empty" } - switch keyType { - case "node": - return fmt.Sprintf("Serialized node (size: %d bytes)", len(value)) - case "write_log", "roots_metadata", "root_updated_nodes", "metadata": - return fmt.Sprintf("CBOR-encoded %s (size: %d bytes)", keyType, len(value)) + if keyType != "node" { + return fmt.Sprintf("MKVS %s (size: %d bytes)", keyType, len(value)) + } + + // Parse MKVS node: 0x00=leaf, 0x01=internal, 0x02=nil + switch value[0] { + case 0x00: // LeafNode: [2-byte keyLen LE][key][4-byte valueLen LE][value] + data := value[1:] + if len(data) < 2 { + return "LeafNode{}" + } + + keyLen := int(binary.LittleEndian.Uint16(data[0:2])) + data = data[2:] + if len(data) < keyLen { + return fmt.Sprintf("LeafNode{keyLen=%d, }", keyLen) + } + + key := data[:keyLen] + data = data[keyLen:] + + // Decode consensus module key prefix (roothash, staking, registry, etc.) + // Consensus modules use single-byte prefixes from oasis-core/go/consensus/tendermint/apps/*/state/state.go + var module string + if len(key) == 0 { + module = "" + } else { + switch key[0] { + // Roothash module (0x20-0x29) + case 0x20: + module = "roothash/runtime_state" + case 0x21: + module = "roothash/params" + case 0x22: + module = "roothash/round_timeout" + case 0x24: + module = "roothash/evidence" + case 0x25: + module = "roothash/state_root" + case 0x26: + module = "roothash/io_root" + case 0x27: + module = "roothash/last_round_results" + case 0x28: + module = "roothash/incoming_msg_queue_meta" + case 0x29: + module = "roothash/incoming_msg_queue" + // Staking module (0x30-0x3F) + case 0x30: + module = "staking/total_supply" + case 0x31: + module = "staking/common_pool" + case 0x32: + module = "staking/last_block_fees" + case 0x33: + module = "staking/governance_deposits" + case 0x34: + module = "staking/accounts" + case 0x35: + module = "staking/delegations" + case 0x36: + module = "staking/debonding_delegations" + case 0x37: + module = "staking/allowances" + case 0x38: + module = "staking/params" + // Registry module (0x40-0x4F) + case 0x40: + module = "registry/entities" + case 0x41: + module = "registry/nodes" + case 0x42: + module = "registry/node_by_consensus" + case 0x43: + module = "registry/runtimes" + case 0x44: + module = "registry/suspended_runtimes" + case 0x45: + module = "registry/params" + case 0x46: + module = "registry/node_status" + // Scheduler module (0x50-0x5F) + case 0x50: + module = "scheduler/params" + case 0x51: + module = "scheduler/committees" + case 0x52: + module = "scheduler/validators" + // Governance module (0x60-0x6F) + case 0x60: + module = "governance/params" + case 0x61: + module = "governance/proposals" + case 0x62: + module = "governance/active_proposals" + case 0x63: + module = "governance/votes" + case 0x64: + module = "governance/pending_upgrades" + // Beacon module (0x70-0x7F) + case 0x70: + module = "beacon/params" + case 0x71: + module = "beacon/future_epoch" + case 0x72: + module = "beacon/epoch" + case 0x73: + module = "beacon/pvss_state" + // Keymanager module (0x80-0x8F) + case 0x80: + module = "keymanager/status" + case 0x81: + module = "keymanager/params" + // Consensus parameters + case 0xF1: + module = "consensus/params" + default: + if key[0] >= 'a' && key[0] <= 'z' { + module = extractModuleName(key) + } else { + module = fmt.Sprintf("0x%02x", key[0]) + } + } + } + + if len(data) < 4 { + return fmt.Sprintf("LeafNode{module=%s, }", module) + } + + valueLen := int(binary.LittleEndian.Uint32(data[0:4])) + data = data[4:] + if len(data) < valueLen { + return fmt.Sprintf("LeafNode{module=%s, valueLen=%d, }", module, valueLen) + } + + leafValue := data[:valueLen] + + // Try CBOR decode + var decoded interface{} + if err := cbor.Unmarshal(leafValue, &decoded); err == nil { + return fmt.Sprintf("LeafNode{module=%s, value=%s}", module, formatCBOR(decoded, valueLen)) + } + + return fmt.Sprintf("LeafNode{module=%s, value=binary(%d bytes)}", module, valueLen) + + case 0x01: // InternalNode: [2-byte labelBits LE][label][leaf/nil marker][hashes] + data := value[1:] + if len(data) < 2 { + return "InternalNode{}" + } + + labelBits := binary.LittleEndian.Uint16(data[0:2]) + data = data[2:] + + labelBytes := (int(labelBits) + 7) / 8 + if len(data) < labelBytes+1 { + return fmt.Sprintf("InternalNode{label=%d bits, }", labelBits) + } + + data = data[labelBytes:] // skip label + + hasLeaf := data[0] == 0x00 + if hasLeaf { + return fmt.Sprintf("InternalNode{label=%d bits, has_leaf=true}", labelBits) + } + + data = data[1:] // skip nil marker + + // Extract child hashes + var left, right string + if len(data) >= 32 { + left = truncateHex(data[:32], 16) + data = data[32:] + } + if len(data) >= 32 { + right = truncateHex(data[:32], 16) + } + + return fmt.Sprintf("InternalNode{label=%d bits, left=%s, right=%s}", labelBits, left, right) + + case 0x02: // NilNode + return "NilNode{}" + default: - return fmt.Sprintf("MKVS %s data (size: %d bytes)", keyType, len(value)) + return fmt.Sprintf("Unknown node prefix 0x%02x (size: %d)", value[0], len(value)) } } @@ -546,27 +775,310 @@ func decodeValueConsensusState(keyType string, value []byte) string { return fmt.Sprintf("Tendermint %s data (size: %d bytes)", keyType, len(value)) } -// analyzeKeyRuntimeMkvs parses runtime-mkvs key (same format as consensus-mkvs) +// analyzeKeyRuntimeMkvs parses runtime-mkvs key +// Key format: [prefix byte][type byte][data...] func analyzeKeyRuntimeMkvs(key []byte) (string, string) { - return analyzeKeyConsensusMkvs(key) + if len(key) < 1 { + return "unknown", fmt.Sprintf("%x", key) + } + + // Check for dbVersion prefix (0x01 or 0x05) + prefixByte := key[0] + var data []byte + + if prefixByte == 0x01 || prefixByte == 0x05 { + if len(key) < 2 { + return "unknown", fmt.Sprintf("%x", key) + } + prefixByte = key[1] + data = key[2:] + } else { + data = key[1:] + } + + switch prefixByte { + case 0x00: + return "node", fmt.Sprintf("node{hash: %s}", truncateHex(data, 16)) + case 0x01: + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + return "write_log", fmt.Sprintf("write_log{v:%d}", version) + } + return "write_log", "write_log{}" + case 0x02: + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + return "roots_metadata", fmt.Sprintf("roots_metadata{v:%d}", version) + } + return "roots_metadata", "roots_metadata{}" + case 0x03: + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{v:%d}", version) + } + return "root_updated_nodes", "root_updated_nodes{}" + case 0x04: + return "metadata", "metadata" + default: + return fmt.Sprintf("unknown_%02x", prefixByte), fmt.Sprintf("%x", key) + } } -// decodeValueRuntimeMkvs decodes runtime MKVS value (same format as consensus MKVS) +// decodeValueRuntimeMkvs decodes runtime MKVS value with inline node parsing func decodeValueRuntimeMkvs(keyType string, value []byte) string { - return decodeValueConsensusMkvs(keyType, value) + if len(value) == 0 { + return "Empty" + } + + if keyType != "node" { + return fmt.Sprintf("MKVS %s (size: %d bytes)", keyType, len(value)) + } + + // Parse MKVS node: 0x00=leaf, 0x01=internal, 0x02=nil + switch value[0] { + case 0x00: // LeafNode: [2-byte keyLen LE][key][4-byte valueLen LE][value] + data := value[1:] + if len(data) < 2 { + return "LeafNode{}" + } + + keyLen := int(binary.LittleEndian.Uint16(data[0:2])) + data = data[2:] + if len(data) < keyLen { + return fmt.Sprintf("LeafNode{keyLen=%d, }", keyLen) + } + + key := data[:keyLen] + data = data[keyLen:] + + // Extract module name from key (runtime-specific: evm, contracts, accounts, etc.) + module := extractModuleName(key) + + if len(data) < 4 { + return fmt.Sprintf("LeafNode{module=%s, }", module) + } + + valueLen := int(binary.LittleEndian.Uint32(data[0:4])) + data = data[4:] + if len(data) < valueLen { + return fmt.Sprintf("LeafNode{module=%s, valueLen=%d, }", module, valueLen) + } + + leafValue := data[:valueLen] + + // Try CBOR decode + var decoded interface{} + if err := cbor.Unmarshal(leafValue, &decoded); err == nil { + return fmt.Sprintf("LeafNode{module=%s, value=%s}", module, formatCBOR(decoded, valueLen)) + } + + return fmt.Sprintf("LeafNode{module=%s, value=binary(%d bytes)}", module, valueLen) + + case 0x01: // InternalNode: [2-byte labelBits LE][label][leaf/nil marker][hashes] + data := value[1:] + if len(data) < 2 { + return "InternalNode{}" + } + + labelBits := binary.LittleEndian.Uint16(data[0:2]) + data = data[2:] + + labelBytes := (int(labelBits) + 7) / 8 + if len(data) < labelBytes+1 { + return fmt.Sprintf("InternalNode{label=%d bits, }", labelBits) + } + + data = data[labelBytes:] // skip label + + hasLeaf := data[0] == 0x00 + if hasLeaf { + return fmt.Sprintf("InternalNode{label=%d bits, has_leaf=true}", labelBits) + } + + data = data[1:] // skip nil marker + + // Extract child hashes + var left, right string + if len(data) >= 32 { + left = truncateHex(data[:32], 16) + data = data[32:] + } + if len(data) >= 32 { + right = truncateHex(data[:32], 16) + } + + return fmt.Sprintf("InternalNode{label=%d bits, left=%s, right=%s}", labelBits, left, right) + + case 0x02: // NilNode + return "NilNode{}" + + default: + return fmt.Sprintf("Unknown node prefix 0x%02x (size: %d)", value[0], len(value)) + } } -// analyzeKeyRuntimeHistory parses runtime-history key (same format as consensus-state) +// extractModuleName extracts ASCII module name from MKVS key +func extractModuleName(key []byte) string { + if len(key) == 0 { + return "" + } + + // Skip leading 0x00 if present + if key[0] == 0x00 && len(key) > 1 { + key = key[1:] + } + + // Find ASCII module name + end := 0 + for i, b := range key { + if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' { + end = i + 1 + } else { + break + } + } + + if end > 0 { + return string(key[:end]) + } + return truncateHex(key, 16) +} + +// formatCBOR formats decoded CBOR value concisely +func formatCBOR(v interface{}, rawLen int) string { + switch val := v.(type) { + case map[interface{}]interface{}: + keys := make([]string, 0, len(val)) + for k := range val { + keys = append(keys, fmt.Sprintf("%v", k)) + } + if len(keys) > 3 { + return fmt.Sprintf("map{%s, +%d}", strings.Join(keys[:3], ","), len(keys)-3) + } + return fmt.Sprintf("map{%s}", strings.Join(keys, ",")) + case []interface{}: + return fmt.Sprintf("array[%d]", len(val)) + case []byte: + return fmt.Sprintf("bytes(%d)", len(val)) + case string: + if len(val) > 20 { + return fmt.Sprintf("%q...", val[:20]) + } + return fmt.Sprintf("%q", val) + case uint64: + return fmt.Sprintf("%d", val) + case int64: + return fmt.Sprintf("%d", val) + case bool: + return fmt.Sprintf("%v", val) + case nil: + return "null" + default: + return fmt.Sprintf("%T(%d bytes)", v, rawLen) + } +} + +// analyzeKeyRuntimeHistory parses runtime-history key using binary prefix format +// Key formats from oasis-core/go/runtime/history/db.go: +// - 0x01: metadata key (1 byte) +// - 0x02 + uint64: block key (9 bytes) - round number in big-endian +// - 0x03 + uint64: round_results key (9 bytes) - round number in big-endian func analyzeKeyRuntimeHistory(key []byte) (string, string) { - return analyzeKeyConsensusState(key) + if len(key) == 0 { + return "unknown", "" + } + + switch key[0] { + case 0x01: + // Metadata key - just the prefix byte + if len(key) == 1 { + return "metadata", "metadata" + } + // Unexpected extra data after metadata prefix + return "metadata", fmt.Sprintf("metadata (extra: %x)", key[1:]) + + case 0x02: + // Block key - prefix + 8-byte round number + if len(key) == 9 { + round := binary.BigEndian.Uint64(key[1:9]) + return "block", fmt.Sprintf("round:%d", round) + } + return "block", fmt.Sprintf("block (malformed, len=%d)", len(key)) + + case 0x03: + // Round results key - prefix + 8-byte round number + if len(key) == 9 { + round := binary.BigEndian.Uint64(key[1:9]) + return "round_results", fmt.Sprintf("round:%d", round) + } + return "round_results", fmt.Sprintf("round_results (malformed, len=%d)", len(key)) + + default: + return "unknown", fmt.Sprintf("%x", key) + } } -// decodeValueRuntimeHistory decodes runtime history value +// decodeValueRuntimeHistory decodes CBOR-encoded runtime history value func decodeValueRuntimeHistory(keyType string, value []byte) string { if len(value) == 0 { return "Empty" } - return fmt.Sprintf("Runtime %s data (size: %d bytes)", keyType, len(value)) + + switch keyType { + case "metadata": + var meta RuntimeHistoryMetadata + if err := cbor.Unmarshal(value, &meta); err != nil { + return fmt.Sprintf("Failed to decode metadata: %v (size: %d)", err, len(value)) + } + runtimeID := truncateHex(meta.RuntimeID, 16) + return fmt.Sprintf("Metadata{version: %d, runtime_id: %s..., last_round: %d, last_consensus_height: %d}", + meta.Version, runtimeID, meta.LastRound, meta.LastConsensusHeight) + + case "block": + var block RuntimeHistoryAnnotatedBlock + if err := cbor.Unmarshal(value, &block); err != nil { + return fmt.Sprintf("Failed to decode block: %v (size: %d)", err, len(value)) + } + if block.Block == nil { + return fmt.Sprintf("AnnotatedBlock{consensus_height: %d, block: nil}", block.Height) + } + h := block.Block.Header + // Format timestamp as human-readable + ts := time.Unix(int64(h.Timestamp), 0).UTC().Format(time.RFC3339) + headerType := headerTypeName(h.HeaderType) + stateRoot := truncateHex(h.StateRoot, 16) + return fmt.Sprintf("AnnotatedBlock{consensus_height: %d, round: %d, timestamp: %s, header_type: %s, state_root: %s...}", + block.Height, h.Round, ts, headerType, stateRoot) + + case "round_results": + var results RuntimeHistoryRoundResults + if err := cbor.Unmarshal(value, &results); err != nil { + return fmt.Sprintf("Failed to decode round_results: %v (size: %d)", err, len(value)) + } + return fmt.Sprintf("RoundResults{messages: %d, good_entities: %d, bad_entities: %d}", + len(results.Messages), len(results.GoodComputeEntities), len(results.BadComputeEntities)) + + default: + return fmt.Sprintf("Runtime %s data (size: %d bytes)", keyType, len(value)) + } +} + +// headerTypeName converts header type byte to string +func headerTypeName(headerType uint8) string { + switch headerType { + case 0: + return "Invalid" + case 1: + return "Normal" + case 2: + return "RoundFailed" + case 3: + return "EpochTransition" + case 4: + return "Suspended" + default: + return fmt.Sprintf("Unknown(%d)", headerType) + } } // isPrintableASCII checks if a string contains only printable ASCII characters From 1f17c2a477c1a01158d5efbc49c2ca6c66caa55e Mon Sep 17 00:00:00 2001 From: gw0 Date: Sat, 22 Nov 2025 11:39:46 +0100 Subject: [PATCH 08/22] badgerdb-sampler: Update dependencies and helper scripts --- badgerdb-sampler/db_v2.go | 2 +- badgerdb-sampler/go_v2.mod | 2 ++ badgerdb-sampler/go_v2.sum | 4 +++ badgerdb-sampler/go_v3.mod | 2 ++ badgerdb-sampler/go_v3.sum | 4 +++ badgerdb-sampler/scripts/mount-snapshots.sh | 7 +++++ .../scripts/run-mainnet-samplers.sh | 28 +++++++++++-------- .../scripts/run-testnet-samplers.sh | 25 ++++++++++------- 8 files changed, 52 insertions(+), 22 deletions(-) diff --git a/badgerdb-sampler/db_v2.go b/badgerdb-sampler/db_v2.go index 5664386..a0c21ad 100644 --- a/badgerdb-sampler/db_v2.go +++ b/badgerdb-sampler/db_v2.go @@ -61,7 +61,7 @@ func openDatabase(path string) (*DB, error) { opts.ReadOnly = false opts.SyncWrites = false opts.NumMemtables = 1 // Minimize memtable usage - opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) + opts.MaxTableSize = 1 << 20 // 1MB table size (minimal) opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 diff --git a/badgerdb-sampler/go_v2.mod b/badgerdb-sampler/go_v2.mod index c493ddd..ae9b17b 100644 --- a/badgerdb-sampler/go_v2.mod +++ b/badgerdb-sampler/go_v2.mod @@ -14,9 +14,11 @@ require ( github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect github.com/dustin/go-humanize v1.0.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.1 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/x448/float16 v0.8.4 // indirect golang.org/x/net v0.0.0-20220726230323-06994584191e // indirect golang.org/x/sys v0.0.0-20220727055044-e65921a090b8 // indirect google.golang.org/protobuf v1.28.0 // indirect diff --git a/badgerdb-sampler/go_v2.sum b/badgerdb-sampler/go_v2.sum index 918a51a..6eaf54d 100644 --- a/badgerdb-sampler/go_v2.sum +++ b/badgerdb-sampler/go_v2.sum @@ -22,6 +22,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -67,6 +69,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/tendermint/tendermint v0.34.21 h1:UiGGnBFHVrZhoQVQ7EfwSOLuCtarqCSsRf8VrklqB7s= github.com/tendermint/tendermint v0.34.21/go.mod h1:XDvfg6U7grcFTDx7VkzxnhazQ/bspGJAn4DZ6DcLLjQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/badgerdb-sampler/go_v3.mod b/badgerdb-sampler/go_v3.mod index c3044af..1db1dea 100644 --- a/badgerdb-sampler/go_v3.mod +++ b/badgerdb-sampler/go_v3.mod @@ -13,6 +13,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -20,6 +21,7 @@ require ( github.com/google/flatbuffers v1.12.1 // indirect github.com/klauspost/compress v1.15.9 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/x448/float16 v0.8.4 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/net v0.0.0-20220726230323-06994584191e // indirect golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect diff --git a/badgerdb-sampler/go_v3.sum b/badgerdb-sampler/go_v3.sum index 4ffd7ff..89410b4 100644 --- a/badgerdb-sampler/go_v3.sum +++ b/badgerdb-sampler/go_v3.sum @@ -31,6 +31,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -106,6 +108,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/tendermint/tendermint v0.34.21 h1:UiGGnBFHVrZhoQVQ7EfwSOLuCtarqCSsRf8VrklqB7s= github.com/tendermint/tendermint v0.34.21/go.mod h1:XDvfg6U7grcFTDx7VkzxnhazQ/bspGJAn4DZ6DcLLjQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/badgerdb-sampler/scripts/mount-snapshots.sh b/badgerdb-sampler/scripts/mount-snapshots.sh index 49a1ef8..e587a46 100755 --- a/badgerdb-sampler/scripts/mount-snapshots.sh +++ b/badgerdb-sampler/scripts/mount-snapshots.sh @@ -1,5 +1,12 @@ #!/bin/bash # Script to mount .tar.zst snapshot archives without extracting +# +# ./mount-snapshots.sh testnet +# +# Manually: +# ./ratarmount -o allow_other,uid=1000,gid=1000 -w "testnet/20220303-20231012/cipher_testnet.overlay" "testnet/20220303-20231012/cipher_testnet.tar.zst" "testnet/20220303-20231012/cipher_testnet.mount" +# fusermount -u "testnet/20220303-20231012/cipher_testnet.mount" && rm -rf "testnet/20220303-20231012/cipher_testnet.mount" "testnet/20220303-20231012/cipher_testnet.overlay" +# set -euo pipefail cd "$(dirname "$0")" diff --git a/badgerdb-sampler/scripts/run-mainnet-samplers.sh b/badgerdb-sampler/scripts/run-mainnet-samplers.sh index 86154f3..217eab8 100755 --- a/badgerdb-sampler/scripts/run-mainnet-samplers.sh +++ b/badgerdb-sampler/scripts/run-mainnet-samplers.sh @@ -9,8 +9,8 @@ OUTPUT="./outputs" mkdir -p "$OUTPUT" # $OUTPUT/mainnet/20201001-20201118/ (BadgerDB v2) -( mkdir -p "$OUTPUT/mainnet-20201001-20201118" +( $SAMPLER_V2 consensus-blockstore-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20201001-20201118/consensus-blockstore-v2.json > $OUTPUT/mainnet-20201001-20201118/consensus-blockstore-v2.log 2>&1 || true $SAMPLER_V2 consensus-evidence-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20201001-20201118/consensus-evidence-v2.json > $OUTPUT/mainnet-20201001-20201118/consensus-evidence-v2.log 2>&1 || true $SAMPLER_V2 consensus-state-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20201001-20201118/consensus-state-v2.json > $OUTPUT/mainnet-20201001-20201118/consensus-state-v2.log 2>&1 || true @@ -18,8 +18,8 @@ $SAMPLER_V2 consensus-mkvs-v2 /snapshots/mainnet/20201001-20201118/consensus.mou ) & # $OUTPUT/mainnet/20201118-20210428/ (BadgerDB v2) -( mkdir -p "$OUTPUT/mainnet-20201118-20210428" +( $SAMPLER_V2 consensus-blockstore-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20201118-20210428/consensus-blockstore-v2.json > $OUTPUT/mainnet-20201118-20210428/consensus-blockstore-v2.log 2>&1 || true $SAMPLER_V2 consensus-evidence-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20201118-20210428/consensus-evidence-v2.json > $OUTPUT/mainnet-20201118-20210428/consensus-evidence-v2.log 2>&1 || true $SAMPLER_V2 consensus-state-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20201118-20210428/consensus-state-v2.json > $OUTPUT/mainnet-20201118-20210428/consensus-state-v2.log 2>&1 || true @@ -27,34 +27,40 @@ $SAMPLER_V2 consensus-mkvs-v2 /snapshots/mainnet/20201118-20210428/consensus.mou ) & # $OUTPUT/mainnet/20210428-20220411/ (BadgerDB v3) -( mkdir -p "$OUTPUT/mainnet-20210428-20220411" +( $SAMPLER_V3 consensus-blockstore-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20210428-20220411/consensus-blockstore-v3.json > $OUTPUT/mainnet-20210428-20220411/consensus-blockstore-v3.log 2>&1 || true $SAMPLER_V3 consensus-evidence-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20210428-20220411/consensus-evidence-v3.json > $OUTPUT/mainnet-20210428-20220411/consensus-evidence-v3.log 2>&1 || true $SAMPLER_V3 consensus-state-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20210428-20220411/consensus-state-v3.json > $OUTPUT/mainnet-20210428-20220411/consensus-state-v3.log 2>&1 || true $SAMPLER_V3 consensus-mkvs-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/consensus-mkvs-v3.json > $OUTPUT/mainnet-20210428-20220411/consensus-mkvs-v3.log 2>&1 || true - +) & +( $SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20210428-20220411/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/cipher-runtime-mkvs-v3.json > $OUTPUT/mainnet-20210428-20220411/cipher-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20210428-20220411/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/emerald-runtime-mkvs-v3.json > $OUTPUT/mainnet-20210428-20220411/emerald-runtime-mkvs-v3.log 2>&1 || true - $SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20210428-20220411/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT/mainnet-20210428-20220411/cipher-runtime-history-v3.json > $OUTPUT/mainnet-20210428-20220411/cipher-runtime-history-v3.log 2>&1 || true +) & +( $SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20210428-20220411/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT/mainnet-20210428-20220411/emerald-runtime-history-v3.json > $OUTPUT/mainnet-20210428-20220411/emerald-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20210428-20220411/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/emerald-runtime-mkvs-v3.json > $OUTPUT/mainnet-20210428-20220411/emerald-runtime-mkvs-v3.log 2>&1 || true ) & # $OUTPUT/mainnet/20220411-20231129/ (BadgerDB v3) -( mkdir -p "$OUTPUT/mainnet-20220411-20231129" +( $SAMPLER_V3 consensus-blockstore-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20220411-20231129/consensus-blockstore-v3.json > $OUTPUT/mainnet-20220411-20231129/consensus-blockstore-v3.log 2>&1 || true $SAMPLER_V3 consensus-evidence-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20220411-20231129/consensus-evidence-v3.json > $OUTPUT/mainnet-20220411-20231129/consensus-evidence-v3.log 2>&1 || true $SAMPLER_V3 consensus-state-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20220411-20231129/consensus-state-v3.json > $OUTPUT/mainnet-20220411-20231129/consensus-state-v3.log 2>&1 || true $SAMPLER_V3 consensus-mkvs-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/consensus-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/consensus-mkvs-v3.log 2>&1 || true - +) & +( $SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/cipher-runtime-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/cipher-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/emerald-runtime-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/emerald-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/sapphire.mount/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-mkvs-v3.log 2>&1 || true - $SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT/mainnet-20220411-20231129/cipher-runtime-history-v3.json > $OUTPUT/mainnet-20220411-20231129/cipher-runtime-history-v3.log 2>&1 || true +) & +( +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/emerald-runtime-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/emerald-runtime-mkvs-v3.log 2>&1 || true $SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT/mainnet-20220411-20231129/emerald-runtime-history-v3.json > $OUTPUT/mainnet-20220411-20231129/emerald-runtime-history-v3.log 2>&1 || true +) & +( +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/sapphire.mount/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-mkvs-v3.log 2>&1 || true $SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/sapphire.mount/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/history.db $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-history-v3.json > $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-history-v3.log 2>&1 || true ) & diff --git a/badgerdb-sampler/scripts/run-testnet-samplers.sh b/badgerdb-sampler/scripts/run-testnet-samplers.sh index 34573f3..d5df5f5 100755 --- a/badgerdb-sampler/scripts/run-testnet-samplers.sh +++ b/badgerdb-sampler/scripts/run-testnet-samplers.sh @@ -9,8 +9,8 @@ OUTPUT="./outputs" mkdir -p "$OUTPUT" # $OUTPUT/testnet/20200915-20201104/ (BadgerDB v2) -( mkdir -p "$OUTPUT/testnet-20200915-20201104" +( $SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20200915-20201104/consensus-blockstore-v2.json > $OUTPUT/testnet-20200915-20201104/consensus-blockstore-v2.log 2>&1 || true $SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20200915-20201104/consensus-evidence-v2.json > $OUTPUT/testnet-20200915-20201104/consensus-evidence-v2.log 2>&1 || true $SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20200915-20201104/consensus-state-v2.json > $OUTPUT/testnet-20200915-20201104/consensus-state-v2.log 2>&1 || true @@ -18,8 +18,8 @@ $SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20200915-20201104/consensus_tes ) & # $OUTPUT/testnet/20201104-20210203/ (BadgerDB v2) -( mkdir -p "$OUTPUT/testnet-20201104-20210203" +( $SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20201104-20210203/consensus-blockstore-v2.json > $OUTPUT/testnet-20201104-20210203/consensus-blockstore-v2.log 2>&1 || true $SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20201104-20210203/consensus-evidence-v2.json > $OUTPUT/testnet-20201104-20210203/consensus-evidence-v2.log 2>&1 || true $SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20201104-20210203/consensus-state-v2.json > $OUTPUT/testnet-20201104-20210203/consensus-state-v2.log 2>&1 || true @@ -27,8 +27,8 @@ $SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20201104-20210203/consensus_tes ) & # $OUTPUT/testnet/20210203-20210324/ (BadgerDB v2) -( mkdir -p "$OUTPUT/testnet-20210203-20210324" +( $SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210203-20210324/consensus-blockstore-v2.json > $OUTPUT/testnet-20210203-20210324/consensus-blockstore-v2.log 2>&1 || true $SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210203-20210324/consensus-evidence-v2.json > $OUTPUT/testnet-20210203-20210324/consensus-evidence-v2.log 2>&1 || true $SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210203-20210324/consensus-state-v2.json > $OUTPUT/testnet-20210203-20210324/consensus-state-v2.log 2>&1 || true @@ -36,8 +36,8 @@ $SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20210203-20210324/consensus_tes ) & # $OUTPUT/testnet/20210324-20210413/ (BadgerDB v2) -( mkdir -p "$OUTPUT/testnet-20210324-20210413" +( $SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210324-20210413/consensus-blockstore-v2.json > $OUTPUT/testnet-20210324-20210413/consensus-blockstore-v2.log 2>&1 || true $SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210324-20210413/consensus-evidence-v2.json > $OUTPUT/testnet-20210324-20210413/consensus-evidence-v2.log 2>&1 || true $SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210324-20210413/consensus-state-v2.json > $OUTPUT/testnet-20210324-20210413/consensus-state-v2.log 2>&1 || true @@ -45,8 +45,8 @@ $SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20210324-20210413/consensus_tes ) & # $OUTPUT/testnet/20210413-20220303/ (BadgerDB v3) -( mkdir -p "$OUTPUT/testnet-20210413-20220303" +( $SAMPLER_V3 consensus-blockstore-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210413-20220303/consensus-blockstore-v3.json > $OUTPUT/testnet-20210413-20220303/consensus-blockstore-v3.log 2>&1 || true $SAMPLER_V3 consensus-evidence-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210413-20220303/consensus-evidence-v3.json > $OUTPUT/testnet-20210413-20220303/consensus-evidence-v3.log 2>&1 || true $SAMPLER_V3 consensus-state-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210413-20220303/consensus-state-v3.json > $OUTPUT/testnet-20210413-20220303/consensus-state-v3.log 2>&1 || true @@ -54,21 +54,26 @@ $SAMPLER_V3 consensus-mkvs-v3 /snapshots/testnet/20210413-20220303/consensus_tes ) & # $OUTPUT/testnet/20220303-20231012/ (BadgerDB v3) -( mkdir -p "$OUTPUT/testnet-20220303-20231012" +( $SAMPLER_V3 consensus-blockstore-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20220303-20231012/consensus-blockstore-v3.json > $OUTPUT/testnet-20220303-20231012/consensus-blockstore-v3.log 2>&1 || true $SAMPLER_V3 consensus-evidence-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20220303-20231012/consensus-evidence-v3.json > $OUTPUT/testnet-20220303-20231012/consensus-evidence-v3.log 2>&1 || true $SAMPLER_V3 consensus-state-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20220303-20231012/consensus-state-v3.json > $OUTPUT/testnet-20220303-20231012/consensus-state-v3.log 2>&1 || true $SAMPLER_V3 consensus-mkvs-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/consensus-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/consensus-mkvs-v3.log 2>&1 || true - +) & +( $SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/cipher_testnet.mount/runtimes/0000000000000000000000000000000000000000000000000000000000000000/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/cipher-runtime-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/cipher-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/emerald_testnet.mount/runtimes/00000000000000000000000000000000000000000000000072c8215e60d5bca7/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/emerald-runtime-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/emerald-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/sapphire_testnet.mount/runtimes/000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/sapphire-runtime-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/sapphire-runtime-mkvs-v3.log 2>&1 || true - $SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/cipher_testnet.mount/runtimes/0000000000000000000000000000000000000000000000000000000000000000/history.db $OUTPUT/testnet-20220303-20231012/cipher-runtime-history-v3.json > $OUTPUT/testnet-20220303-20231012/cipher-runtime-history-v3.log 2>&1 || true +) & +( +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/emerald_testnet.mount/runtimes/00000000000000000000000000000000000000000000000072c8215e60d5bca7/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/emerald-runtime-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/emerald-runtime-mkvs-v3.log 2>&1 || true $SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/emerald_testnet.mount/runtimes/00000000000000000000000000000000000000000000000072c8215e60d5bca7/history.db $OUTPUT/testnet-20220303-20231012/emerald-runtime-history-v3.json > $OUTPUT/testnet-20220303-20231012/emerald-runtime-history-v3.log 2>&1 || true +) & +( +$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/sapphire_testnet.mount/runtimes/000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/sapphire-runtime-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/sapphire-runtime-mkvs-v3.log 2>&1 || true $SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/sapphire_testnet.mount/runtimes/000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c/history.db $OUTPUT/testnet-20220303-20231012/sapphire-runtime-history-v3.json > $OUTPUT/testnet-20220303-20231012/sapphire-runtime-history-v3.log 2>&1 || true ) & echo "All samplers launched in background!" +echo "> ps aux | grep badgerdb-sampler" ps aux | grep badgerdb-sampler From b38fba2d4ee708d2d5b390dc42a76c8c99672a52 Mon Sep 17 00:00:00 2001 From: gw0 Date: Sat, 22 Nov 2025 12:01:31 +0100 Subject: [PATCH 09/22] badgerdb-sampler: Split code into consensus and runtime analysis --- badgerdb-sampler/decode_consensus.go | 486 +++++++++++++++ badgerdb-sampler/decode_runtime.go | 304 +++++++++ badgerdb-sampler/main.go | 882 +-------------------------- badgerdb-sampler/utils.go | 97 +++ 4 files changed, 896 insertions(+), 873 deletions(-) create mode 100644 badgerdb-sampler/decode_consensus.go create mode 100644 badgerdb-sampler/decode_runtime.go create mode 100644 badgerdb-sampler/utils.go diff --git a/badgerdb-sampler/decode_consensus.go b/badgerdb-sampler/decode_consensus.go new file mode 100644 index 0000000..2fcd262 --- /dev/null +++ b/badgerdb-sampler/decode_consensus.go @@ -0,0 +1,486 @@ +package main + +import ( + "encoding/binary" + "fmt" + "strconv" + "strings" + "time" + + "github.com/fxamacker/cbor/v2" + "github.com/gogo/protobuf/proto" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + tmstore "github.com/tendermint/tendermint/proto/tendermint/store" +) + +// decodeKeyConsensusBlockstore parses consensus-blockstore key and returns key type and decoded representation +func decodeKeyConsensusBlockstore(key []byte) (string, string) { + if len(key) < 2 { + return "unknown", fmt.Sprintf("%x", key) + } + + // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) + if key[0] != 0x01 { + return "unknown", fmt.Sprintf("%x", key) + } + + // Parse ASCII key after 0x01 prefix + asciiKey := string(key[1:]) + + // blockStore state key + if asciiKey == "blockStore" { + return "blockstore_state", "blockStore" + } + + // Parse structured keys: "H:{height}", "P:{height}:{part}", "C:{height}", "SC:{height}", "BH:{hash}" + if strings.HasPrefix(asciiKey, "H:") { + return "block_meta", asciiKey + } + if strings.HasPrefix(asciiKey, "P:") { + return "block_part", asciiKey + } + if strings.HasPrefix(asciiKey, "C:") { + return "block_commit", asciiKey + } + if strings.HasPrefix(asciiKey, "SC:") { + return "seen_commit", asciiKey + } + if strings.HasPrefix(asciiKey, "BH:") { + return "block_hash", asciiKey + } + + return "unknown", asciiKey +} + +// decodeValueConsensusBlockstore attempts to decode protobuf value based on key type +// Returns: (decoded_string, timestamp_unix) +func decodeValueConsensusBlockstore(keyType string, value []byte) (string, int64) { + switch keyType { + case "blockstore_state": + var state tmstore.BlockStoreState + if err := proto.Unmarshal(value, &state); err == nil { + return fmt.Sprintf("BlockStoreState{base: %d, height: %d}", state.Base, state.Height), 0 + } + + case "block_meta": + var meta tmproto.BlockMeta + if err := proto.Unmarshal(value, &meta); err == nil { + // Extract timestamp + tsUnix := int64(0) + ts := "" + if meta.Header.Time.Unix() > 0 { + tsUnix = meta.Header.Time.Unix() + ts = meta.Header.Time.Format(time.RFC3339) + } + + // Format AppHash and ChainID + appHash := "" + if len(meta.Header.AppHash) >= 8 { + appHash = fmt.Sprintf("%x", meta.Header.AppHash[:8]) + } else if len(meta.Header.AppHash) > 0 { + appHash = fmt.Sprintf("%x", meta.Header.AppHash) + } + + chainID := meta.Header.ChainID + if len(chainID) > 20 { + chainID = chainID[:20] + "..." + } + + return fmt.Sprintf("BlockMeta{height: %d, time: %s, chain: %s, num_txs: %d, app_hash: %s}", + meta.Header.Height, ts, chainID, meta.NumTxs, appHash), tsUnix + } + + case "block_part": + var part tmproto.Part + if err := proto.Unmarshal(value, &part); err == nil { + return fmt.Sprintf("Part{index: %d, bytes_size: %d, proof_total: %d}", + part.Index, len(part.Bytes), part.Proof.Total), 0 + } + + case "block_commit": + var commit tmproto.Commit + if err := proto.Unmarshal(value, &commit); err == nil { + return fmt.Sprintf("Commit{height: %d, round: %d, signatures: %d}", + commit.Height, commit.Round, len(commit.Signatures)), 0 + } + + case "seen_commit": + var commit tmproto.Commit + if err := proto.Unmarshal(value, &commit); err == nil { + return fmt.Sprintf("Commit{height: %d, round: %d, signatures: %d}", + commit.Height, commit.Round, len(commit.Signatures)), 0 + } + + case "block_hash": + // Block hash values are plain strings containing height numbers + height, err := strconv.ParseInt(string(value), 10, 64) + if err == nil { + return fmt.Sprintf("Hash{height: %d}", height), 0 + } + + default: + return fmt.Sprintf("Unknown type (size: %d bytes)", len(value)), 0 + } + + return fmt.Sprintf("Failed to decode blockstore (type: %s, size: %d bytes)", keyType, len(value)), 0 +} + +// decodeKeyConsensusEvidence parses consensus-evidence key and returns key type and decoded representation +func decodeKeyConsensusEvidence(key []byte) (string, string) { + if len(key) < 2 { + return "unknown", fmt.Sprintf("%x", key) + } + + // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) + if key[0] != 0x01 { + return "unknown", fmt.Sprintf("%x", key) + } + + // Evidence DB is typically empty or uses simple key patterns + return fmt.Sprintf("type_%02x", key[1]), fmt.Sprintf("%x", key[1:]) +} + +// decodeValueConsensusEvidence attempts to decode evidence value +func decodeValueConsensusEvidence(keyType string, value []byte) string { + if len(value) == 0 { + return "Empty" + } + + // Try to decode as DuplicateVoteEvidence + var evidence tmproto.DuplicateVoteEvidence + if err := proto.Unmarshal(value, &evidence); err == nil { + return fmt.Sprintf("DuplicateVoteEvidence{vote_a_height: %d, vote_b_height: %d}", + evidence.VoteA.Height, evidence.VoteB.Height) + } + + return fmt.Sprintf("Evidence (type: %s, size: %d bytes)", keyType, len(value)) +} + +// decodeKeyConsensusMkvs parses consensus-mkvs key and returns key type and decoded representation +// FIXED: Handles both old format (no dbVersion prefix) and new format (0x01 or 0x05 dbVersion prefix) +// MKVS keys from oasis-core/go/storage/mkvs/db/badger/badger.go +func decodeKeyConsensusMkvs(key []byte) (string, string) { + if len(key) < 1 { + return "unknown", fmt.Sprintf("%x", key) + } + + // Check if key has dbVersion prefix (0x01 or 0x05) + // Old format: type byte directly at key[0] + // New format: 0x01 or 0x05 at key[0], type byte at key[1] + prefixByte := key[0] + var data []byte + + if prefixByte == 0x01 || prefixByte == 0x05 { + // New format with dbVersion prefix + if len(key) < 2 { + return "unknown", fmt.Sprintf("%x", key) + } + prefixByte = key[1] + data = key[2:] + } else { + // Old format without dbVersion prefix (BadgerDB v2 MKVS) + data = key[1:] + } + + switch prefixByte { + case 0x00: + // nodeKeyFmt: hash.Hash (32 bytes) + return "node", fmt.Sprintf("node{hash: %x....}", truncateBytes(data, 8)) + case 0x01: + // writeLogKeyFmt: uint64(version) + TypedHash(new root) + TypedHash(old root) + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + if len(data) >= 8+33 { // version + first TypedHash (33 bytes in v3+) + rootType := data[8] + rootHash := data[9 : 9+32] + return "write_log", fmt.Sprintf("write_log{v:%d, new_root_type:%d, hash: %x...}", version, rootType, truncateBytes(rootHash, 4)) + } + return "write_log", fmt.Sprintf("write_log{v:%d}", version) + } + return "write_log", "write_log{}" + case 0x02: + // rootsMetadataKeyFmt: uint64(version) + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + return "roots_metadata", fmt.Sprintf("roots_metadata{v:%d}", version) + } + return "roots_metadata", "roots_metadata{}" + case 0x03: + // rootUpdatedNodesKeyFmt: uint64(version) + TypedHash(root) + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + if len(data) >= 8+33 { // version + TypedHash + rootType := data[8] + rootHash := data[9 : 9+32] + return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{v:%d, root_type:%d, hash:%x...}", version, rootType, truncateBytes(rootHash, 4)) + } + return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{v:%d}", version) + } + return "root_updated_nodes", "root_updated_nodes{}" + case 0x04: + // metadataKeyFmt: no additional data + return "metadata", "metadata" + case 0x05: + // multipartRestoreNodeLogKeyFmt: TypedHash (33 bytes) + if len(data) >= 33 { + rootType := data[0] + rootHash := data[1:33] + return "multipart_restore_log", fmt.Sprintf("multipart_restore_log{type:%d, hash:%x}", rootType, truncateBytes(rootHash, 8)) + } + return "multipart_restore_log", fmt.Sprintf("multipart_restore_log{hash:%x}", truncateBytes(data, 8)) + case 0x06: + // rootNodeKeyFmt: TypedHash (33 bytes) + if len(data) >= 33 { + rootType := data[0] + rootHash := data[1:33] + return "root_node", fmt.Sprintf("root_node{type:%d, hash:%x}", rootType, truncateBytes(rootHash, 8)) + } + return "root_node", fmt.Sprintf("root_node{hash:%x}", truncateBytes(data, 8)) + default: + return fmt.Sprintf("unknown_%02x", prefixByte), fmt.Sprintf("unknown_%02x:%x", prefixByte, truncateBytes(data, 8)) + } +} + +// decodeValueConsensusMkvs decodes consensus MKVS value with inline node parsing +func decodeValueConsensusMkvs(keyType string, value []byte) string { + if len(value) == 0 { + return "Empty" + } + + if keyType != "node" { + return fmt.Sprintf("MKVS %s (size: %d bytes)", keyType, len(value)) + } + + // Parse MKVS node: 0x00=leaf, 0x01=internal, 0x02=nil + switch value[0] { + case 0x00: // LeafNode: [2-byte keyLen LE][key][4-byte valueLen LE][value] + data := value[1:] + if len(data) < 2 { + return "LeafNode{}" + } + + keyLen := int(binary.LittleEndian.Uint16(data[0:2])) + data = data[2:] + if len(data) < keyLen { + return fmt.Sprintf("LeafNode{keyLen=%d, }", keyLen) + } + + key := data[:keyLen] + data = data[keyLen:] + + // Decode consensus module key prefix (roothash, staking, registry, etc.) + // Consensus modules use single-byte prefixes from oasis-core/go/consensus/tendermint/apps/*/state/state.go + var module string + if len(key) == 0 { + module = "" + } else { + switch key[0] { + // Roothash module (0x20-0x29) + case 0x20: + module = "roothash/runtime_state" + case 0x21: + module = "roothash/params" + case 0x22: + module = "roothash/round_timeout" + case 0x24: + module = "roothash/evidence" + case 0x25: + module = "roothash/state_root" + case 0x26: + module = "roothash/io_root" + case 0x27: + module = "roothash/last_round_results" + case 0x28: + module = "roothash/incoming_msg_queue_meta" + case 0x29: + module = "roothash/incoming_msg_queue" + // Staking module (0x30-0x3F) + case 0x30: + module = "staking/total_supply" + case 0x31: + module = "staking/common_pool" + case 0x32: + module = "staking/last_block_fees" + case 0x33: + module = "staking/governance_deposits" + case 0x34: + module = "staking/accounts" + case 0x35: + module = "staking/delegations" + case 0x36: + module = "staking/debonding_delegations" + case 0x37: + module = "staking/allowances" + case 0x38: + module = "staking/params" + // Registry module (0x40-0x4F) + case 0x40: + module = "registry/entities" + case 0x41: + module = "registry/nodes" + case 0x42: + module = "registry/node_by_consensus" + case 0x43: + module = "registry/runtimes" + case 0x44: + module = "registry/suspended_runtimes" + case 0x45: + module = "registry/params" + case 0x46: + module = "registry/node_status" + // Scheduler module (0x50-0x5F) + case 0x50: + module = "scheduler/params" + case 0x51: + module = "scheduler/committees" + case 0x52: + module = "scheduler/validators" + // Governance module (0x60-0x6F) + case 0x60: + module = "governance/params" + case 0x61: + module = "governance/proposals" + case 0x62: + module = "governance/active_proposals" + case 0x63: + module = "governance/votes" + case 0x64: + module = "governance/pending_upgrades" + // Beacon module (0x70-0x7F) + case 0x70: + module = "beacon/params" + case 0x71: + module = "beacon/future_epoch" + case 0x72: + module = "beacon/epoch" + case 0x73: + module = "beacon/pvss_state" + // Keymanager module (0x80-0x8F) + case 0x80: + module = "keymanager/status" + case 0x81: + module = "keymanager/params" + // Consensus parameters + case 0xF1: + module = "consensus/params" + default: + if key[0] >= 'a' && key[0] <= 'z' { + module = extractModuleName(key) + } else { + module = fmt.Sprintf("0x%02x", key[0]) + } + } + } + + if len(data) < 4 { + return fmt.Sprintf("LeafNode{module=%s, }", module) + } + + valueLen := int(binary.LittleEndian.Uint32(data[0:4])) + data = data[4:] + if len(data) < valueLen { + return fmt.Sprintf("LeafNode{module=%s, valueLen=%d, }", module, valueLen) + } + + leafValue := data[:valueLen] + + // Try CBOR decode + var decoded interface{} + if err := cbor.Unmarshal(leafValue, &decoded); err == nil { + return fmt.Sprintf("LeafNode{module=%s, value=%s}", module, formatCBOR(decoded, valueLen)) + } + + return fmt.Sprintf("LeafNode{module=%s, value=binary(%d bytes)}", module, valueLen) + + case 0x01: // InternalNode: [2-byte labelBits LE][label][leaf/nil marker][hashes] + data := value[1:] + if len(data) < 2 { + return "InternalNode{}" + } + + labelBits := binary.LittleEndian.Uint16(data[0:2]) + data = data[2:] + + labelBytes := (int(labelBits) + 7) / 8 + if len(data) < labelBytes+1 { + return fmt.Sprintf("InternalNode{label=%d bits, }", labelBits) + } + + data = data[labelBytes:] // skip label + + hasLeaf := data[0] == 0x00 + if hasLeaf { + return fmt.Sprintf("InternalNode{label=%d bits, has_leaf=true}", labelBits) + } + + data = data[1:] // skip nil marker + + // Extract child hashes + var left, right string + if len(data) >= 32 { + left = truncateHex(data[:32], 16) + data = data[32:] + } + if len(data) >= 32 { + right = truncateHex(data[:32], 16) + } + + return fmt.Sprintf("InternalNode{label=%d bits, left=%s, right=%s}", labelBits, left, right) + + case 0x02: // NilNode + return "NilNode{}" + + default: + return fmt.Sprintf("Unknown node prefix 0x%02x (size: %d)", value[0], len(value)) + } +} + +// decodeKeyConsensusState parses consensus-state key and returns key type and decoded representation +func decodeKeyConsensusState(key []byte) (string, string) { + if len(key) < 2 { + return "unknown", fmt.Sprintf("%x", key) + } + + // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) + if key[0] != 0x01 { + return "unknown", fmt.Sprintf("%x", key) + } + + // Parse ASCII key after 0x01 prefix + decoded := string(key[1:]) + + // Check if it's printable ASCII + if !isPrintableASCII(decoded) { + return "binary", fmt.Sprintf("%x", key) + } + + // Extract key type prefix before colon + colonIdx := strings.IndexByte(decoded, ':') + if colonIdx != -1 { + prefix := decoded[:colonIdx] + switch prefix { + case "abciResponsesKey": + return "abci_responses", decoded + case "consensusParamsKey": + return "consensus_params", decoded + case "validatorsKey": + return "validators", decoded + case "stateKey": + return "state", decoded + case "genesisDoc": + return "genesis", decoded + default: + return prefix, decoded + } + } + + return "text_key", decoded +} + +// decodeValueConsensusState decodes state value +func decodeValueConsensusState(keyType string, value []byte) string { + if len(value) == 0 { + return "Empty" + } + return fmt.Sprintf("Tendermint %s data (size: %d bytes)", keyType, len(value)) +} diff --git a/badgerdb-sampler/decode_runtime.go b/badgerdb-sampler/decode_runtime.go new file mode 100644 index 0000000..eeb149f --- /dev/null +++ b/badgerdb-sampler/decode_runtime.go @@ -0,0 +1,304 @@ +package main + +import ( + "encoding/binary" + "fmt" + "time" + + "github.com/fxamacker/cbor/v2" +) + +// Runtime history CBOR types (from oasis-core/go/runtime/history/db.go) + +// RuntimeHistoryMetadata represents the metadata stored in runtime history DB +type RuntimeHistoryMetadata struct { + RuntimeID []byte `cbor:"runtime_id"` + Version uint64 `cbor:"version"` + LastConsensusHeight int64 `cbor:"last_consensus_height"` + LastRound uint64 `cbor:"last_round"` +} + +// RuntimeHistoryAnnotatedBlock represents an annotated block in runtime history +type RuntimeHistoryAnnotatedBlock struct { + Height int64 `cbor:"consensus_height"` + Block *RuntimeHistoryBlock `cbor:"block"` +} + +// RuntimeHistoryBlock represents a runtime block +type RuntimeHistoryBlock struct { + Header RuntimeHistoryBlockHeader `cbor:"header"` +} + +// RuntimeHistoryBlockHeader represents a runtime block header +type RuntimeHistoryBlockHeader struct { + Version uint16 `cbor:"version"` + Namespace []byte `cbor:"namespace"` + Round uint64 `cbor:"round"` + Timestamp uint64 `cbor:"timestamp"` + HeaderType uint8 `cbor:"header_type"` + PreviousHash []byte `cbor:"previous_hash"` + IORoot []byte `cbor:"io_root"` + StateRoot []byte `cbor:"state_root"` + MessagesHash []byte `cbor:"messages_hash"` + InMessagesHash []byte `cbor:"in_msgs_hash"` +} + +// RuntimeHistoryRoundResults represents round results in runtime history +type RuntimeHistoryRoundResults struct { + Messages []RuntimeHistoryMessageEvent `cbor:"messages,omitempty"` + GoodComputeEntities [][]byte `cbor:"good_compute_entities,omitempty"` + BadComputeEntities [][]byte `cbor:"bad_compute_entities,omitempty"` +} + +// RuntimeHistoryMessageEvent represents a message event +type RuntimeHistoryMessageEvent struct { + Module string `cbor:"module,omitempty"` + Code uint32 `cbor:"code,omitempty"` + Index uint32 `cbor:"index,omitempty"` + Result cbor.RawMessage `cbor:"result,omitempty"` +} + +// decodeKeyRuntimeMkvs parses runtime-mkvs key +// Key format: [prefix byte][type byte][data...] +func decodeKeyRuntimeMkvs(key []byte) (string, string) { + if len(key) < 1 { + return "unknown", fmt.Sprintf("%x", key) + } + + // Check for dbVersion prefix (0x01 or 0x05) + prefixByte := key[0] + var data []byte + + if prefixByte == 0x01 || prefixByte == 0x05 { + if len(key) < 2 { + return "unknown", fmt.Sprintf("%x", key) + } + prefixByte = key[1] + data = key[2:] + } else { + data = key[1:] + } + + switch prefixByte { + case 0x00: + return "node", fmt.Sprintf("node{hash: %s}", truncateHex(data, 16)) + case 0x01: + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + return "write_log", fmt.Sprintf("write_log{v:%d}", version) + } + return "write_log", "write_log{}" + case 0x02: + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + return "roots_metadata", fmt.Sprintf("roots_metadata{v:%d}", version) + } + return "roots_metadata", "roots_metadata{}" + case 0x03: + if len(data) >= 8 { + version := binary.BigEndian.Uint64(data[0:8]) + return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{v:%d}", version) + } + return "root_updated_nodes", "root_updated_nodes{}" + case 0x04: + return "metadata", "metadata" + default: + return fmt.Sprintf("unknown_%02x", prefixByte), fmt.Sprintf("%x", key) + } +} + +// decodeValueRuntimeMkvs decodes runtime MKVS value with inline node parsing +func decodeValueRuntimeMkvs(keyType string, value []byte) string { + if len(value) == 0 { + return "Empty" + } + + if keyType != "node" { + return fmt.Sprintf("MKVS %s (size: %d bytes)", keyType, len(value)) + } + + // Parse MKVS node: 0x00=leaf, 0x01=internal, 0x02=nil + switch value[0] { + case 0x00: // LeafNode: [2-byte keyLen LE][key][4-byte valueLen LE][value] + data := value[1:] + if len(data) < 2 { + return "LeafNode{}" + } + + keyLen := int(binary.LittleEndian.Uint16(data[0:2])) + data = data[2:] + if len(data) < keyLen { + return fmt.Sprintf("LeafNode{keyLen=%d, }", keyLen) + } + + key := data[:keyLen] + data = data[keyLen:] + + // Extract module name from key (runtime-specific: evm, contracts, accounts, etc.) + module := extractModuleName(key) + + if len(data) < 4 { + return fmt.Sprintf("LeafNode{module=%s, }", module) + } + + valueLen := int(binary.LittleEndian.Uint32(data[0:4])) + data = data[4:] + if len(data) < valueLen { + return fmt.Sprintf("LeafNode{module=%s, valueLen=%d, }", module, valueLen) + } + + leafValue := data[:valueLen] + + // Try CBOR decode + var decoded interface{} + if err := cbor.Unmarshal(leafValue, &decoded); err == nil { + return fmt.Sprintf("LeafNode{module=%s, value=%s}", module, formatCBOR(decoded, valueLen)) + } + + return fmt.Sprintf("LeafNode{module=%s, value=binary(%d bytes)}", module, valueLen) + + case 0x01: // InternalNode: [2-byte labelBits LE][label][leaf/nil marker][hashes] + data := value[1:] + if len(data) < 2 { + return "InternalNode{}" + } + + labelBits := binary.LittleEndian.Uint16(data[0:2]) + data = data[2:] + + labelBytes := (int(labelBits) + 7) / 8 + if len(data) < labelBytes+1 { + return fmt.Sprintf("InternalNode{label=%d bits, }", labelBits) + } + + data = data[labelBytes:] // skip label + + hasLeaf := data[0] == 0x00 + if hasLeaf { + return fmt.Sprintf("InternalNode{label=%d bits, has_leaf=true}", labelBits) + } + + data = data[1:] // skip nil marker + + // Extract child hashes + var left, right string + if len(data) >= 32 { + left = truncateHex(data[:32], 16) + data = data[32:] + } + if len(data) >= 32 { + right = truncateHex(data[:32], 16) + } + + return fmt.Sprintf("InternalNode{label=%d bits, left=%s, right=%s}", labelBits, left, right) + + case 0x02: // NilNode + return "NilNode{}" + + default: + return fmt.Sprintf("Unknown node prefix 0x%02x (size: %d)", value[0], len(value)) + } +} + +// decodeKeyRuntimeHistory parses runtime-history key using binary prefix format +// Key formats from oasis-core/go/runtime/history/db.go: +// - 0x01: metadata key (1 byte) +// - 0x02 + uint64: block key (9 bytes) - round number in big-endian +// - 0x03 + uint64: round_results key (9 bytes) - round number in big-endian +func decodeKeyRuntimeHistory(key []byte) (string, string) { + if len(key) == 0 { + return "unknown", "" + } + + switch key[0] { + case 0x01: + // Metadata key - just the prefix byte + if len(key) == 1 { + return "metadata", "metadata" + } + // Unexpected extra data after metadata prefix + return "metadata", fmt.Sprintf("metadata (extra: %x)", key[1:]) + + case 0x02: + // Block key - prefix + 8-byte round number + if len(key) == 9 { + round := binary.BigEndian.Uint64(key[1:9]) + return "block", fmt.Sprintf("round:%d", round) + } + return "block", fmt.Sprintf("block (malformed, len=%d)", len(key)) + + case 0x03: + // Round results key - prefix + 8-byte round number + if len(key) == 9 { + round := binary.BigEndian.Uint64(key[1:9]) + return "round_results", fmt.Sprintf("round:%d", round) + } + return "round_results", fmt.Sprintf("round_results (malformed, len=%d)", len(key)) + + default: + return "unknown", fmt.Sprintf("%x", key) + } +} + +// decodeValueRuntimeHistory decodes CBOR-encoded runtime history value +func decodeValueRuntimeHistory(keyType string, value []byte) string { + if len(value) == 0 { + return "Empty" + } + + switch keyType { + case "metadata": + var meta RuntimeHistoryMetadata + if err := cbor.Unmarshal(value, &meta); err != nil { + return fmt.Sprintf("Failed to decode metadata: %v (size: %d)", err, len(value)) + } + runtimeID := truncateHex(meta.RuntimeID, 16) + return fmt.Sprintf("Metadata{version: %d, runtime_id: %s..., last_round: %d, last_consensus_height: %d}", + meta.Version, runtimeID, meta.LastRound, meta.LastConsensusHeight) + + case "block": + var block RuntimeHistoryAnnotatedBlock + if err := cbor.Unmarshal(value, &block); err != nil { + return fmt.Sprintf("Failed to decode block: %v (size: %d)", err, len(value)) + } + if block.Block == nil { + return fmt.Sprintf("AnnotatedBlock{consensus_height: %d, block: nil}", block.Height) + } + h := block.Block.Header + // Format timestamp as human-readable + ts := time.Unix(int64(h.Timestamp), 0).UTC().Format(time.RFC3339) + headerType := headerTypeName(h.HeaderType) + stateRoot := truncateHex(h.StateRoot, 16) + return fmt.Sprintf("AnnotatedBlock{consensus_height: %d, round: %d, timestamp: %s, header_type: %s, state_root: %s...}", + block.Height, h.Round, ts, headerType, stateRoot) + + case "round_results": + var results RuntimeHistoryRoundResults + if err := cbor.Unmarshal(value, &results); err != nil { + return fmt.Sprintf("Failed to decode round_results: %v (size: %d)", err, len(value)) + } + return fmt.Sprintf("RoundResults{messages: %d, good_entities: %d, bad_entities: %d}", + len(results.Messages), len(results.GoodComputeEntities), len(results.BadComputeEntities)) + + default: + return fmt.Sprintf("Runtime %s data (size: %d bytes)", keyType, len(value)) + } +} + +// headerTypeName converts header type byte to string +func headerTypeName(headerType uint8) string { + switch headerType { + case 0: + return "Invalid" + case 1: + return "Normal" + case 2: + return "RoundFailed" + case 3: + return "EpochTransition" + case 4: + return "Suspended" + default: + return fmt.Sprintf("Unknown(%d)", headerType) + } +} diff --git a/badgerdb-sampler/main.go b/badgerdb-sampler/main.go index 8b2de5b..bf05c27 100644 --- a/badgerdb-sampler/main.go +++ b/badgerdb-sampler/main.go @@ -1,7 +1,6 @@ package main import ( - "encoding/binary" "encoding/json" "fmt" "log" @@ -10,11 +9,6 @@ import ( "strconv" "strings" "time" - - "github.com/fxamacker/cbor/v2" - "github.com/gogo/protobuf/proto" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" - tmstore "github.com/tendermint/tendermint/proto/tendermint/store" ) // BadgerVersion is set by build tags @@ -43,56 +37,6 @@ type DBStats struct { Samples []Sample `json:"samples"` } -// Runtime history CBOR types (from oasis-core/go/runtime/history/db.go) - -// RuntimeHistoryMetadata represents the metadata stored in runtime history DB -type RuntimeHistoryMetadata struct { - RuntimeID []byte `cbor:"runtime_id"` - Version uint64 `cbor:"version"` - LastConsensusHeight int64 `cbor:"last_consensus_height"` - LastRound uint64 `cbor:"last_round"` -} - -// RuntimeHistoryAnnotatedBlock represents an annotated block in runtime history -type RuntimeHistoryAnnotatedBlock struct { - Height int64 `cbor:"consensus_height"` - Block *RuntimeHistoryBlock `cbor:"block"` -} - -// RuntimeHistoryBlock represents a runtime block -type RuntimeHistoryBlock struct { - Header RuntimeHistoryBlockHeader `cbor:"header"` -} - -// RuntimeHistoryBlockHeader represents a runtime block header -type RuntimeHistoryBlockHeader struct { - Version uint16 `cbor:"version"` - Namespace []byte `cbor:"namespace"` - Round uint64 `cbor:"round"` - Timestamp uint64 `cbor:"timestamp"` - HeaderType uint8 `cbor:"header_type"` - PreviousHash []byte `cbor:"previous_hash"` - IORoot []byte `cbor:"io_root"` - StateRoot []byte `cbor:"state_root"` - MessagesHash []byte `cbor:"messages_hash"` - InMessagesHash []byte `cbor:"in_msgs_hash"` -} - -// RuntimeHistoryRoundResults represents round results in runtime history -type RuntimeHistoryRoundResults struct { - Messages []RuntimeHistoryMessageEvent `cbor:"messages,omitempty"` - GoodComputeEntities [][]byte `cbor:"good_compute_entities,omitempty"` - BadComputeEntities [][]byte `cbor:"bad_compute_entities,omitempty"` -} - -// RuntimeHistoryMessageEvent represents a message event -type RuntimeHistoryMessageEvent struct { - Module string `cbor:"module,omitempty"` - Code uint32 `cbor:"code,omitempty"` - Index uint32 `cbor:"index,omitempty"` - Result cbor.RawMessage `cbor:"result,omitempty"` -} - func main() { if len(os.Args) < 3 { fmt.Fprintf(os.Stderr, "Usage: %s [output-json] [max-samples]\n", os.Args[0]) @@ -204,8 +148,8 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { return db.View(func(txn *Txn) error { opts := DefaultIteratorOptions opts.PrefetchValues = true - opts.PrefetchSize = 100 // Optimal prefetch size for BadgerDB - opts.AllVersions = false // Only latest version of each key + opts.PrefetchSize = 100 // Optimal prefetch size for BadgerDB + opts.AllVersions = false // Only latest version of each key it := txn.NewIterator(opts) defer it.Close() @@ -234,30 +178,30 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { DecodedKey: "", } - // Analyze key based on database type + // Decode key based on database type switch stats.DatabaseType { case "consensus-blockstore": - keyType, decodedKey := analyzeKeyConsensusBlockstore(key) + keyType, decodedKey := decodeKeyConsensusBlockstore(key) sample.KeyType = keyType sample.DecodedKey = decodedKey case "consensus-evidence": - keyType, decodedKey := analyzeKeyConsensusEvidence(key) + keyType, decodedKey := decodeKeyConsensusEvidence(key) sample.KeyType = keyType sample.DecodedKey = decodedKey case "consensus-mkvs": - keyType, decodedKey := analyzeKeyConsensusMkvs(key) + keyType, decodedKey := decodeKeyConsensusMkvs(key) sample.KeyType = keyType sample.DecodedKey = decodedKey case "consensus-state": - keyType, decodedKey := analyzeKeyConsensusState(key) + keyType, decodedKey := decodeKeyConsensusState(key) sample.KeyType = keyType sample.DecodedKey = decodedKey case "runtime-mkvs": - keyType, decodedKey := analyzeKeyRuntimeMkvs(key) + keyType, decodedKey := decodeKeyRuntimeMkvs(key) sample.KeyType = keyType sample.DecodedKey = decodedKey case "runtime-history": - keyType, decodedKey := analyzeKeyRuntimeHistory(key) + keyType, decodedKey := decodeKeyRuntimeHistory(key) sample.KeyType = keyType sample.DecodedKey = decodedKey } @@ -302,811 +246,3 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { return nil }) } - -// analyzeKeyConsensusBlockstore parses consensus-blockstore key and returns key type and decoded representation -func analyzeKeyConsensusBlockstore(key []byte) (string, string) { - if len(key) < 2 { - return "unknown", fmt.Sprintf("%x", key) - } - - // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) - if key[0] != 0x01 { - return "unknown", fmt.Sprintf("%x", key) - } - - // Parse ASCII key after 0x01 prefix - asciiKey := string(key[1:]) - - // blockStore state key - if asciiKey == "blockStore" { - return "blockstore_state", "blockStore" - } - - // Parse structured keys: "H:{height}", "P:{height}:{part}", "C:{height}", "SC:{height}", "BH:{hash}" - if strings.HasPrefix(asciiKey, "H:") { - return "block_meta", asciiKey - } - if strings.HasPrefix(asciiKey, "P:") { - return "block_part", asciiKey - } - if strings.HasPrefix(asciiKey, "C:") { - return "block_commit", asciiKey - } - if strings.HasPrefix(asciiKey, "SC:") { - return "seen_commit", asciiKey - } - if strings.HasPrefix(asciiKey, "BH:") { - return "block_hash", asciiKey - } - - return "unknown", asciiKey -} - -// decodeValueConsensusBlockstore attempts to decode protobuf value based on key type -// Returns: (decoded_string, timestamp_unix) -func decodeValueConsensusBlockstore(keyType string, value []byte) (string, int64) { - switch keyType { - case "blockstore_state": - var state tmstore.BlockStoreState - if err := proto.Unmarshal(value, &state); err == nil { - return fmt.Sprintf("BlockStoreState{base: %d, height: %d}", state.Base, state.Height), 0 - } - - case "block_meta": - var meta tmproto.BlockMeta - if err := proto.Unmarshal(value, &meta); err == nil { - // Extract timestamp - tsUnix := int64(0) - ts := "" - if meta.Header.Time.Unix() > 0 { - tsUnix = meta.Header.Time.Unix() - ts = meta.Header.Time.Format(time.RFC3339) - } - - // Format AppHash and ChainID - appHash := "" - if len(meta.Header.AppHash) >= 8 { - appHash = fmt.Sprintf("%x", meta.Header.AppHash[:8]) - } else if len(meta.Header.AppHash) > 0 { - appHash = fmt.Sprintf("%x", meta.Header.AppHash) - } - - chainID := meta.Header.ChainID - if len(chainID) > 20 { - chainID = chainID[:20] + "..." - } - - return fmt.Sprintf("BlockMeta{height: %d, time: %s, chain: %s, num_txs: %d, app_hash: %s}", - meta.Header.Height, ts, chainID, meta.NumTxs, appHash), tsUnix - } - - case "block_part": - var part tmproto.Part - if err := proto.Unmarshal(value, &part); err == nil { - return fmt.Sprintf("Part{index: %d, bytes_size: %d, proof_total: %d}", - part.Index, len(part.Bytes), part.Proof.Total), 0 - } - - case "block_commit": - var commit tmproto.Commit - if err := proto.Unmarshal(value, &commit); err == nil { - return fmt.Sprintf("Commit{height: %d, round: %d, signatures: %d}", - commit.Height, commit.Round, len(commit.Signatures)), 0 - } - - case "seen_commit": - var commit tmproto.Commit - if err := proto.Unmarshal(value, &commit); err == nil { - return fmt.Sprintf("Commit{height: %d, round: %d, signatures: %d}", - commit.Height, commit.Round, len(commit.Signatures)), 0 - } - - case "block_hash": - // Block hash values are plain strings containing height numbers - height, err := strconv.ParseInt(string(value), 10, 64) - if err == nil { - return fmt.Sprintf("Hash{height: %d}", height), 0 - } - - default: - return fmt.Sprintf("Unknown type (size: %d bytes)", len(value)), 0 - } - - return fmt.Sprintf("Failed to decode blockstore (type: %s, size: %d bytes)", keyType, len(value)), 0 -} - -// analyzeKeyConsensusEvidence parses consensus-evidence key and returns key type and decoded representation -func analyzeKeyConsensusEvidence(key []byte) (string, string) { - if len(key) < 2 { - return "unknown", fmt.Sprintf("%x", key) - } - - // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) - if key[0] != 0x01 { - return "unknown", fmt.Sprintf("%x", key) - } - - // Evidence DB is typically empty or uses simple key patterns - return fmt.Sprintf("type_%02x", key[1]), fmt.Sprintf("%x", key[1:]) -} - -// decodeValueConsensusEvidence attempts to decode evidence value -func decodeValueConsensusEvidence(keyType string, value []byte) string { - if len(value) == 0 { - return "Empty" - } - - // Try to decode as DuplicateVoteEvidence - var evidence tmproto.DuplicateVoteEvidence - if err := proto.Unmarshal(value, &evidence); err == nil { - return fmt.Sprintf("DuplicateVoteEvidence{vote_a_height: %d, vote_b_height: %d}", - evidence.VoteA.Height, evidence.VoteB.Height) - } - - return fmt.Sprintf("Evidence (type: %s, size: %d bytes)", keyType, len(value)) -} - -// analyzeKeyConsensusMkvs parses consensus-mkvs key and returns key type and decoded representation -// FIXED: Handles both old format (no dbVersion prefix) and new format (0x01 or 0x05 dbVersion prefix) -// MKVS keys from oasis-core/go/storage/mkvs/db/badger/badger.go -func analyzeKeyConsensusMkvs(key []byte) (string, string) { - if len(key) < 1 { - return "unknown", fmt.Sprintf("%x", key) - } - - // Check if key has dbVersion prefix (0x01 or 0x05) - // Old format: type byte directly at key[0] - // New format: 0x01 or 0x05 at key[0], type byte at key[1] - prefixByte := key[0] - var data []byte - - if prefixByte == 0x01 || prefixByte == 0x05 { - // New format with dbVersion prefix - if len(key) < 2 { - return "unknown", fmt.Sprintf("%x", key) - } - prefixByte = key[1] - data = key[2:] - } else { - // Old format without dbVersion prefix (BadgerDB v2 MKVS) - data = key[1:] - } - - switch prefixByte { - case 0x00: - // nodeKeyFmt: hash.Hash (32 bytes) - return "node", fmt.Sprintf("node{hash: %x....}", truncateBytes(data, 8)) - case 0x01: - // writeLogKeyFmt: uint64(version) + TypedHash(new root) + TypedHash(old root) - if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - if len(data) >= 8+33 { // version + first TypedHash (33 bytes in v3+) - rootType := data[8] - rootHash := data[9:9+32] - return "write_log", fmt.Sprintf("write_log{v:%d, new_root_type:%d, hash: %x...}", version, rootType, truncateBytes(rootHash, 4)) - } - return "write_log", fmt.Sprintf("write_log{v:%d}", version) - } - return "write_log", "write_log{}" - case 0x02: - // rootsMetadataKeyFmt: uint64(version) - if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - return "roots_metadata", fmt.Sprintf("roots_metadata{v:%d}", version) - } - return "roots_metadata", "roots_metadata{}" - case 0x03: - // rootUpdatedNodesKeyFmt: uint64(version) + TypedHash(root) - if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - if len(data) >= 8+33 { // version + TypedHash - rootType := data[8] - rootHash := data[9:9+32] - return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{v:%d, root_type:%d, hash:%x...}", version, rootType, truncateBytes(rootHash, 4)) - } - return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{v:%d}", version) - } - return "root_updated_nodes", "root_updated_nodes{}" - case 0x04: - // metadataKeyFmt: no additional data - return "metadata", "metadata" - case 0x05: - // multipartRestoreNodeLogKeyFmt: TypedHash (33 bytes) - if len(data) >= 33 { - rootType := data[0] - rootHash := data[1:33] - return "multipart_restore_log", fmt.Sprintf("multipart_restore_log{type:%d, hash:%x}", rootType, truncateBytes(rootHash, 8)) - } - return "multipart_restore_log", fmt.Sprintf("multipart_restore_log{hash:%x}", truncateBytes(data, 8)) - case 0x06: - // rootNodeKeyFmt: TypedHash (33 bytes) - if len(data) >= 33 { - rootType := data[0] - rootHash := data[1:33] - return "root_node", fmt.Sprintf("root_node{type:%d, hash:%x}", rootType, truncateBytes(rootHash, 8)) - } - return "root_node", fmt.Sprintf("root_node{hash:%x}", truncateBytes(data, 8)) - default: - return fmt.Sprintf("unknown_%02x", prefixByte), fmt.Sprintf("unknown_%02x:%x", prefixByte, truncateBytes(data, 8)) - } -} - -// decodeValueConsensusMkvs decodes consensus MKVS value with inline node parsing -func decodeValueConsensusMkvs(keyType string, value []byte) string { - if len(value) == 0 { - return "Empty" - } - - if keyType != "node" { - return fmt.Sprintf("MKVS %s (size: %d bytes)", keyType, len(value)) - } - - // Parse MKVS node: 0x00=leaf, 0x01=internal, 0x02=nil - switch value[0] { - case 0x00: // LeafNode: [2-byte keyLen LE][key][4-byte valueLen LE][value] - data := value[1:] - if len(data) < 2 { - return "LeafNode{}" - } - - keyLen := int(binary.LittleEndian.Uint16(data[0:2])) - data = data[2:] - if len(data) < keyLen { - return fmt.Sprintf("LeafNode{keyLen=%d, }", keyLen) - } - - key := data[:keyLen] - data = data[keyLen:] - - // Decode consensus module key prefix (roothash, staking, registry, etc.) - // Consensus modules use single-byte prefixes from oasis-core/go/consensus/tendermint/apps/*/state/state.go - var module string - if len(key) == 0 { - module = "" - } else { - switch key[0] { - // Roothash module (0x20-0x29) - case 0x20: - module = "roothash/runtime_state" - case 0x21: - module = "roothash/params" - case 0x22: - module = "roothash/round_timeout" - case 0x24: - module = "roothash/evidence" - case 0x25: - module = "roothash/state_root" - case 0x26: - module = "roothash/io_root" - case 0x27: - module = "roothash/last_round_results" - case 0x28: - module = "roothash/incoming_msg_queue_meta" - case 0x29: - module = "roothash/incoming_msg_queue" - // Staking module (0x30-0x3F) - case 0x30: - module = "staking/total_supply" - case 0x31: - module = "staking/common_pool" - case 0x32: - module = "staking/last_block_fees" - case 0x33: - module = "staking/governance_deposits" - case 0x34: - module = "staking/accounts" - case 0x35: - module = "staking/delegations" - case 0x36: - module = "staking/debonding_delegations" - case 0x37: - module = "staking/allowances" - case 0x38: - module = "staking/params" - // Registry module (0x40-0x4F) - case 0x40: - module = "registry/entities" - case 0x41: - module = "registry/nodes" - case 0x42: - module = "registry/node_by_consensus" - case 0x43: - module = "registry/runtimes" - case 0x44: - module = "registry/suspended_runtimes" - case 0x45: - module = "registry/params" - case 0x46: - module = "registry/node_status" - // Scheduler module (0x50-0x5F) - case 0x50: - module = "scheduler/params" - case 0x51: - module = "scheduler/committees" - case 0x52: - module = "scheduler/validators" - // Governance module (0x60-0x6F) - case 0x60: - module = "governance/params" - case 0x61: - module = "governance/proposals" - case 0x62: - module = "governance/active_proposals" - case 0x63: - module = "governance/votes" - case 0x64: - module = "governance/pending_upgrades" - // Beacon module (0x70-0x7F) - case 0x70: - module = "beacon/params" - case 0x71: - module = "beacon/future_epoch" - case 0x72: - module = "beacon/epoch" - case 0x73: - module = "beacon/pvss_state" - // Keymanager module (0x80-0x8F) - case 0x80: - module = "keymanager/status" - case 0x81: - module = "keymanager/params" - // Consensus parameters - case 0xF1: - module = "consensus/params" - default: - if key[0] >= 'a' && key[0] <= 'z' { - module = extractModuleName(key) - } else { - module = fmt.Sprintf("0x%02x", key[0]) - } - } - } - - if len(data) < 4 { - return fmt.Sprintf("LeafNode{module=%s, }", module) - } - - valueLen := int(binary.LittleEndian.Uint32(data[0:4])) - data = data[4:] - if len(data) < valueLen { - return fmt.Sprintf("LeafNode{module=%s, valueLen=%d, }", module, valueLen) - } - - leafValue := data[:valueLen] - - // Try CBOR decode - var decoded interface{} - if err := cbor.Unmarshal(leafValue, &decoded); err == nil { - return fmt.Sprintf("LeafNode{module=%s, value=%s}", module, formatCBOR(decoded, valueLen)) - } - - return fmt.Sprintf("LeafNode{module=%s, value=binary(%d bytes)}", module, valueLen) - - case 0x01: // InternalNode: [2-byte labelBits LE][label][leaf/nil marker][hashes] - data := value[1:] - if len(data) < 2 { - return "InternalNode{}" - } - - labelBits := binary.LittleEndian.Uint16(data[0:2]) - data = data[2:] - - labelBytes := (int(labelBits) + 7) / 8 - if len(data) < labelBytes+1 { - return fmt.Sprintf("InternalNode{label=%d bits, }", labelBits) - } - - data = data[labelBytes:] // skip label - - hasLeaf := data[0] == 0x00 - if hasLeaf { - return fmt.Sprintf("InternalNode{label=%d bits, has_leaf=true}", labelBits) - } - - data = data[1:] // skip nil marker - - // Extract child hashes - var left, right string - if len(data) >= 32 { - left = truncateHex(data[:32], 16) - data = data[32:] - } - if len(data) >= 32 { - right = truncateHex(data[:32], 16) - } - - return fmt.Sprintf("InternalNode{label=%d bits, left=%s, right=%s}", labelBits, left, right) - - case 0x02: // NilNode - return "NilNode{}" - - default: - return fmt.Sprintf("Unknown node prefix 0x%02x (size: %d)", value[0], len(value)) - } -} - -// analyzeKeyConsensusState parses consensus-state key and returns key type and decoded representation -func analyzeKeyConsensusState(key []byte) (string, string) { - if len(key) < 2 { - return "unknown", fmt.Sprintf("%x", key) - } - - // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) - if key[0] != 0x01 { - return "unknown", fmt.Sprintf("%x", key) - } - - // Parse ASCII key after 0x01 prefix - decoded := string(key[1:]) - - // Check if it's printable ASCII - if !isPrintableASCII(decoded) { - return "binary", fmt.Sprintf("%x", key) - } - - // Extract key type prefix before colon - colonIdx := strings.IndexByte(decoded, ':') - if colonIdx != -1 { - prefix := decoded[:colonIdx] - switch prefix { - case "abciResponsesKey": - return "abci_responses", decoded - case "consensusParamsKey": - return "consensus_params", decoded - case "validatorsKey": - return "validators", decoded - case "stateKey": - return "state", decoded - case "genesisDoc": - return "genesis", decoded - default: - return prefix, decoded - } - } - - return "text_key", decoded -} - -// decodeValueConsensusState decodes state value -func decodeValueConsensusState(keyType string, value []byte) string { - if len(value) == 0 { - return "Empty" - } - return fmt.Sprintf("Tendermint %s data (size: %d bytes)", keyType, len(value)) -} - -// analyzeKeyRuntimeMkvs parses runtime-mkvs key -// Key format: [prefix byte][type byte][data...] -func analyzeKeyRuntimeMkvs(key []byte) (string, string) { - if len(key) < 1 { - return "unknown", fmt.Sprintf("%x", key) - } - - // Check for dbVersion prefix (0x01 or 0x05) - prefixByte := key[0] - var data []byte - - if prefixByte == 0x01 || prefixByte == 0x05 { - if len(key) < 2 { - return "unknown", fmt.Sprintf("%x", key) - } - prefixByte = key[1] - data = key[2:] - } else { - data = key[1:] - } - - switch prefixByte { - case 0x00: - return "node", fmt.Sprintf("node{hash: %s}", truncateHex(data, 16)) - case 0x01: - if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - return "write_log", fmt.Sprintf("write_log{v:%d}", version) - } - return "write_log", "write_log{}" - case 0x02: - if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - return "roots_metadata", fmt.Sprintf("roots_metadata{v:%d}", version) - } - return "roots_metadata", "roots_metadata{}" - case 0x03: - if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{v:%d}", version) - } - return "root_updated_nodes", "root_updated_nodes{}" - case 0x04: - return "metadata", "metadata" - default: - return fmt.Sprintf("unknown_%02x", prefixByte), fmt.Sprintf("%x", key) - } -} - -// decodeValueRuntimeMkvs decodes runtime MKVS value with inline node parsing -func decodeValueRuntimeMkvs(keyType string, value []byte) string { - if len(value) == 0 { - return "Empty" - } - - if keyType != "node" { - return fmt.Sprintf("MKVS %s (size: %d bytes)", keyType, len(value)) - } - - // Parse MKVS node: 0x00=leaf, 0x01=internal, 0x02=nil - switch value[0] { - case 0x00: // LeafNode: [2-byte keyLen LE][key][4-byte valueLen LE][value] - data := value[1:] - if len(data) < 2 { - return "LeafNode{}" - } - - keyLen := int(binary.LittleEndian.Uint16(data[0:2])) - data = data[2:] - if len(data) < keyLen { - return fmt.Sprintf("LeafNode{keyLen=%d, }", keyLen) - } - - key := data[:keyLen] - data = data[keyLen:] - - // Extract module name from key (runtime-specific: evm, contracts, accounts, etc.) - module := extractModuleName(key) - - if len(data) < 4 { - return fmt.Sprintf("LeafNode{module=%s, }", module) - } - - valueLen := int(binary.LittleEndian.Uint32(data[0:4])) - data = data[4:] - if len(data) < valueLen { - return fmt.Sprintf("LeafNode{module=%s, valueLen=%d, }", module, valueLen) - } - - leafValue := data[:valueLen] - - // Try CBOR decode - var decoded interface{} - if err := cbor.Unmarshal(leafValue, &decoded); err == nil { - return fmt.Sprintf("LeafNode{module=%s, value=%s}", module, formatCBOR(decoded, valueLen)) - } - - return fmt.Sprintf("LeafNode{module=%s, value=binary(%d bytes)}", module, valueLen) - - case 0x01: // InternalNode: [2-byte labelBits LE][label][leaf/nil marker][hashes] - data := value[1:] - if len(data) < 2 { - return "InternalNode{}" - } - - labelBits := binary.LittleEndian.Uint16(data[0:2]) - data = data[2:] - - labelBytes := (int(labelBits) + 7) / 8 - if len(data) < labelBytes+1 { - return fmt.Sprintf("InternalNode{label=%d bits, }", labelBits) - } - - data = data[labelBytes:] // skip label - - hasLeaf := data[0] == 0x00 - if hasLeaf { - return fmt.Sprintf("InternalNode{label=%d bits, has_leaf=true}", labelBits) - } - - data = data[1:] // skip nil marker - - // Extract child hashes - var left, right string - if len(data) >= 32 { - left = truncateHex(data[:32], 16) - data = data[32:] - } - if len(data) >= 32 { - right = truncateHex(data[:32], 16) - } - - return fmt.Sprintf("InternalNode{label=%d bits, left=%s, right=%s}", labelBits, left, right) - - case 0x02: // NilNode - return "NilNode{}" - - default: - return fmt.Sprintf("Unknown node prefix 0x%02x (size: %d)", value[0], len(value)) - } -} - -// extractModuleName extracts ASCII module name from MKVS key -func extractModuleName(key []byte) string { - if len(key) == 0 { - return "" - } - - // Skip leading 0x00 if present - if key[0] == 0x00 && len(key) > 1 { - key = key[1:] - } - - // Find ASCII module name - end := 0 - for i, b := range key { - if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' { - end = i + 1 - } else { - break - } - } - - if end > 0 { - return string(key[:end]) - } - return truncateHex(key, 16) -} - -// formatCBOR formats decoded CBOR value concisely -func formatCBOR(v interface{}, rawLen int) string { - switch val := v.(type) { - case map[interface{}]interface{}: - keys := make([]string, 0, len(val)) - for k := range val { - keys = append(keys, fmt.Sprintf("%v", k)) - } - if len(keys) > 3 { - return fmt.Sprintf("map{%s, +%d}", strings.Join(keys[:3], ","), len(keys)-3) - } - return fmt.Sprintf("map{%s}", strings.Join(keys, ",")) - case []interface{}: - return fmt.Sprintf("array[%d]", len(val)) - case []byte: - return fmt.Sprintf("bytes(%d)", len(val)) - case string: - if len(val) > 20 { - return fmt.Sprintf("%q...", val[:20]) - } - return fmt.Sprintf("%q", val) - case uint64: - return fmt.Sprintf("%d", val) - case int64: - return fmt.Sprintf("%d", val) - case bool: - return fmt.Sprintf("%v", val) - case nil: - return "null" - default: - return fmt.Sprintf("%T(%d bytes)", v, rawLen) - } -} - -// analyzeKeyRuntimeHistory parses runtime-history key using binary prefix format -// Key formats from oasis-core/go/runtime/history/db.go: -// - 0x01: metadata key (1 byte) -// - 0x02 + uint64: block key (9 bytes) - round number in big-endian -// - 0x03 + uint64: round_results key (9 bytes) - round number in big-endian -func analyzeKeyRuntimeHistory(key []byte) (string, string) { - if len(key) == 0 { - return "unknown", "" - } - - switch key[0] { - case 0x01: - // Metadata key - just the prefix byte - if len(key) == 1 { - return "metadata", "metadata" - } - // Unexpected extra data after metadata prefix - return "metadata", fmt.Sprintf("metadata (extra: %x)", key[1:]) - - case 0x02: - // Block key - prefix + 8-byte round number - if len(key) == 9 { - round := binary.BigEndian.Uint64(key[1:9]) - return "block", fmt.Sprintf("round:%d", round) - } - return "block", fmt.Sprintf("block (malformed, len=%d)", len(key)) - - case 0x03: - // Round results key - prefix + 8-byte round number - if len(key) == 9 { - round := binary.BigEndian.Uint64(key[1:9]) - return "round_results", fmt.Sprintf("round:%d", round) - } - return "round_results", fmt.Sprintf("round_results (malformed, len=%d)", len(key)) - - default: - return "unknown", fmt.Sprintf("%x", key) - } -} - -// decodeValueRuntimeHistory decodes CBOR-encoded runtime history value -func decodeValueRuntimeHistory(keyType string, value []byte) string { - if len(value) == 0 { - return "Empty" - } - - switch keyType { - case "metadata": - var meta RuntimeHistoryMetadata - if err := cbor.Unmarshal(value, &meta); err != nil { - return fmt.Sprintf("Failed to decode metadata: %v (size: %d)", err, len(value)) - } - runtimeID := truncateHex(meta.RuntimeID, 16) - return fmt.Sprintf("Metadata{version: %d, runtime_id: %s..., last_round: %d, last_consensus_height: %d}", - meta.Version, runtimeID, meta.LastRound, meta.LastConsensusHeight) - - case "block": - var block RuntimeHistoryAnnotatedBlock - if err := cbor.Unmarshal(value, &block); err != nil { - return fmt.Sprintf("Failed to decode block: %v (size: %d)", err, len(value)) - } - if block.Block == nil { - return fmt.Sprintf("AnnotatedBlock{consensus_height: %d, block: nil}", block.Height) - } - h := block.Block.Header - // Format timestamp as human-readable - ts := time.Unix(int64(h.Timestamp), 0).UTC().Format(time.RFC3339) - headerType := headerTypeName(h.HeaderType) - stateRoot := truncateHex(h.StateRoot, 16) - return fmt.Sprintf("AnnotatedBlock{consensus_height: %d, round: %d, timestamp: %s, header_type: %s, state_root: %s...}", - block.Height, h.Round, ts, headerType, stateRoot) - - case "round_results": - var results RuntimeHistoryRoundResults - if err := cbor.Unmarshal(value, &results); err != nil { - return fmt.Sprintf("Failed to decode round_results: %v (size: %d)", err, len(value)) - } - return fmt.Sprintf("RoundResults{messages: %d, good_entities: %d, bad_entities: %d}", - len(results.Messages), len(results.GoodComputeEntities), len(results.BadComputeEntities)) - - default: - return fmt.Sprintf("Runtime %s data (size: %d bytes)", keyType, len(value)) - } -} - -// headerTypeName converts header type byte to string -func headerTypeName(headerType uint8) string { - switch headerType { - case 0: - return "Invalid" - case 1: - return "Normal" - case 2: - return "RoundFailed" - case 3: - return "EpochTransition" - case 4: - return "Suspended" - default: - return fmt.Sprintf("Unknown(%d)", headerType) - } -} - -// isPrintableASCII checks if a string contains only printable ASCII characters -func isPrintableASCII(s string) bool { - if len(s) == 0 { - return false - } - for _, r := range s { - if r < 32 || r > 126 { - return false - } - } - return true -} - -// truncateBytes returns first n bytes or all bytes if shorter -func truncateBytes(data []byte, n int) []byte { - if len(data) <= n { - return data - } - return data[:n] -} - -// truncateHex converts bytes to hex and truncates if too long -func truncateHex(data []byte, maxLen int) string { - hex := fmt.Sprintf("%x", data) - if len(hex) > maxLen { - return hex[:maxLen] + "..." - } - return hex -} diff --git a/badgerdb-sampler/utils.go b/badgerdb-sampler/utils.go new file mode 100644 index 0000000..33377d2 --- /dev/null +++ b/badgerdb-sampler/utils.go @@ -0,0 +1,97 @@ +package main + +import ( + "fmt" + "strings" +) + +// extractModuleName extracts ASCII module name from MKVS key +func extractModuleName(key []byte) string { + if len(key) == 0 { + return "" + } + + // Skip leading 0x00 if present + if key[0] == 0x00 && len(key) > 1 { + key = key[1:] + } + + // Find ASCII module name + end := 0 + for i, b := range key { + if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' { + end = i + 1 + } else { + break + } + } + + if end > 0 { + return string(key[:end]) + } + return truncateHex(key, 16) +} + +// formatCBOR formats decoded CBOR value concisely +func formatCBOR(v interface{}, rawLen int) string { + switch val := v.(type) { + case map[interface{}]interface{}: + keys := make([]string, 0, len(val)) + for k := range val { + keys = append(keys, fmt.Sprintf("%v", k)) + } + if len(keys) > 3 { + return fmt.Sprintf("map{%s, +%d}", strings.Join(keys[:3], ","), len(keys)-3) + } + return fmt.Sprintf("map{%s}", strings.Join(keys, ",")) + case []interface{}: + return fmt.Sprintf("array[%d]", len(val)) + case []byte: + return fmt.Sprintf("bytes(%d)", len(val)) + case string: + if len(val) > 20 { + return fmt.Sprintf("%q...", val[:20]) + } + return fmt.Sprintf("%q", val) + case uint64: + return fmt.Sprintf("%d", val) + case int64: + return fmt.Sprintf("%d", val) + case bool: + return fmt.Sprintf("%v", val) + case nil: + return "null" + default: + return fmt.Sprintf("%T(%d bytes)", v, rawLen) + } +} + +// isPrintableASCII checks if a string contains only printable ASCII characters +func isPrintableASCII(s string) bool { + if len(s) == 0 { + return false + } + for _, r := range s { + if r < 32 || r > 126 { + return false + } + } + return true +} + +// truncateBytes returns first n bytes or all bytes if shorter +func truncateBytes(data []byte, n int) []byte { + if len(data) <= n { + return data + } + return data[:n] +} + +// truncateHex converts bytes to hex and truncates if too long +func truncateHex(data []byte, maxLen int) string { + hex := fmt.Sprintf("%x", data) + if len(hex) > maxLen { + return hex[:maxLen] + "..." + } + return hex +} From 7c7ea0ffedab5aefdd008dfd5d608b64fc7096b8 Mon Sep 17 00:00:00 2001 From: gw0 Date: Mon, 24 Nov 2025 10:04:52 +0100 Subject: [PATCH 10/22] badgerdb-sampler: Extend runtime decoding --- badgerdb-sampler/decode_runtime.go | 79 +++++++++++++++---- badgerdb-sampler/utils.go | 119 ++++++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 16 deletions(-) diff --git a/badgerdb-sampler/decode_runtime.go b/badgerdb-sampler/decode_runtime.go index eeb149f..e69d6f8 100644 --- a/badgerdb-sampler/decode_runtime.go +++ b/badgerdb-sampler/decode_runtime.go @@ -58,6 +58,21 @@ type RuntimeHistoryMessageEvent struct { Result cbor.RawMessage `cbor:"result,omitempty"` } +// RuntimeInputArtifacts represents input transaction artifacts stored in IO tree +// From oasis-core/go/runtime/transaction/transaction.go +type RuntimeInputArtifacts struct { + _ struct{} `cbor:",toarray"` + Input []byte + BatchOrder uint32 +} + +// RuntimeOutputArtifacts represents output transaction artifacts stored in IO tree +// From oasis-core/go/runtime/transaction/transaction.go +type RuntimeOutputArtifacts struct { + _ struct{} `cbor:",toarray"` + Output []byte +} + // decodeKeyRuntimeMkvs parses runtime-mkvs key // Key format: [prefix byte][type byte][data...] func decodeKeyRuntimeMkvs(key []byte) (string, string) { @@ -84,20 +99,20 @@ func decodeKeyRuntimeMkvs(key []byte) (string, string) { return "node", fmt.Sprintf("node{hash: %s}", truncateHex(data, 16)) case 0x01: if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - return "write_log", fmt.Sprintf("write_log{v:%d}", version) + height := binary.BigEndian.Uint64(data[0:8]) + return "write_log", fmt.Sprintf("write_log{height:%d}", height) } return "write_log", "write_log{}" case 0x02: if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - return "roots_metadata", fmt.Sprintf("roots_metadata{v:%d}", version) + height := binary.BigEndian.Uint64(data[0:8]) + return "roots_metadata", fmt.Sprintf("roots_metadata{height:%d}", height) } return "roots_metadata", "roots_metadata{}" case 0x03: if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{v:%d}", version) + height := binary.BigEndian.Uint64(data[0:8]) + return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{height:%d}", height) } return "root_updated_nodes", "root_updated_nodes{}" case 0x04: @@ -149,13 +164,9 @@ func decodeValueRuntimeMkvs(keyType string, value []byte) string { leafValue := data[:valueLen] - // Try CBOR decode - var decoded interface{} - if err := cbor.Unmarshal(leafValue, &decoded); err == nil { - return fmt.Sprintf("LeafNode{module=%s, value=%s}", module, formatCBOR(decoded, valueLen)) - } - - return fmt.Sprintf("LeafNode{module=%s, value=binary(%d bytes)}", module, valueLen) + // Try to decode based on module type + valueDesc := decodeLeafValue(module, key, leafValue) + return fmt.Sprintf("LeafNode{module=%s, value=%s}", module, valueDesc) case 0x01: // InternalNode: [2-byte labelBits LE][label][leaf/nil marker][hashes] data := value[1:] @@ -302,3 +313,45 @@ func headerTypeName(headerType uint8) string { return fmt.Sprintf("Unknown(%d)", headerType) } } + +// decodeLeafValue decodes MKVS leaf value based on module type +func decodeLeafValue(module string, key []byte, value []byte) string { + // Check for IO transaction artifacts + if len(module) > 5 && module[:5] == "io_tx" { + // Determine artifact kind from key + if len(key) >= 34 { + kind := key[33] + if kind == 1 { + // Input artifact + var ia RuntimeInputArtifacts + if err := cbor.Unmarshal(value, &ia); err == nil { + return fmt.Sprintf("RuntimeInputArtifacts{input_size=%d, batch_order=%d}", len(ia.Input), ia.BatchOrder) + } + } else if kind == 2 { + // Output artifact + var oa RuntimeOutputArtifacts + if err := cbor.Unmarshal(value, &oa); err == nil { + return fmt.Sprintf("RuntimeOutputArtifacts{output_size=%d}", len(oa.Output)) + } + } + } + } + + // Check for IO event tags + if len(module) > 8 && module[:8] == "io_event" { + // Event tag value is typically CBOR + var decoded interface{} + if err := cbor.Unmarshal(value, &decoded); err == nil { + return fmt.Sprintf("event_value=%s", formatCBOR(decoded, len(value))) + } + return fmt.Sprintf("event_value=binary(%d bytes)", len(value)) + } + + // Try CBOR decode for regular state keys + var decoded interface{} + if err := cbor.Unmarshal(value, &decoded); err == nil { + return formatCBOR(decoded, len(value)) + } + + return fmt.Sprintf("binary(%d bytes)", len(value)) +} diff --git a/badgerdb-sampler/utils.go b/badgerdb-sampler/utils.go index 33377d2..8cd0b10 100644 --- a/badgerdb-sampler/utils.go +++ b/badgerdb-sampler/utils.go @@ -5,7 +5,7 @@ import ( "strings" ) -// extractModuleName extracts ASCII module name from MKVS key +// extractModuleName extracts module name from MKVS leaf key and returns module:subtype description func extractModuleName(key []byte) string { if len(key) == 0 { return "" @@ -16,7 +16,43 @@ func extractModuleName(key []byte) string { key = key[1:] } - // Find ASCII module name + // Check for IO Tree prefixes + switch key[0] { + case 'T': // Transaction artifact prefix (0x54) + // Key format: 'T' + tx_hash (32 bytes) + artifact_kind (1 byte) + if len(key) >= 34 { + kind := key[33] + kindStr := "unknown" + if kind == 1 { + kindStr = "input" + } else if kind == 2 { + kindStr = "output" + } + return fmt.Sprintf("io_tx:%s (hash=%s)", kindStr, truncateHex(key[1:33], 16)) + } + return fmt.Sprintf("io_tx (hash=%s)", truncateHex(key[1:], 16)) + + case 'E': // Event tag prefix (0x45) + // Key format: 'E' + tag_key (variable, module name) + tx_hash (32 bytes) + if len(key) > 33 { + tagKey := key[1 : len(key)-32] + // Extract module name from tag key + tagModule := "" + for _, b := range tagKey { + if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' { + tagModule += string(b) + } else { + break + } + } + if tagModule != "" { + return fmt.Sprintf("io_event:%s", tagModule) + } + } + return "io_event" + } + + // Regular state key: module_name + sub_prefix + data end := 0 for i, b := range key { if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' { @@ -27,11 +63,88 @@ func extractModuleName(key []byte) string { } if end > 0 { - return string(key[:end]) + moduleName := string(key[:end]) + subKey := key[end:] + return describeModuleKey(moduleName, subKey) } return truncateHex(key, 16) } +// describeModuleKey returns module name with sub-key type description +func describeModuleKey(module string, subKey []byte) string { + if len(subKey) == 0 { + return module + } + + prefix := subKey[0] + var subType string + + switch module { + case "evm": + switch prefix { + case 0x01: + subType = "code" + case 0x02: + subType = "storage" + case 0x03: + subType = "block_hash" + case 0x04: + subType = "confidential_storage" + } + case "accounts": + switch prefix { + case 0x01: + subType = "account" + case 0x02: + subType = "balance" + case 0x03: + subType = "total_supply" + } + case "contracts": + switch prefix { + case 0x01: + subType = "next_code_id" + case 0x02: + subType = "next_instance_id" + case 0x03: + subType = "code_info" + case 0x04: + subType = "instance_info" + case 0x05: + subType = "instance_state" + case 0xFF: + subType = "code" + } + case "core": + switch prefix { + case 0x01: + subType = "metadata" + case 0x02: + subType = "message_handlers" + case 0x03: + subType = "last_epoch" + case 0x04: + subType = "dynamic_min_gas_price" + } + case "consensus_accounts": + switch prefix { + case 0x01: + subType = "delegations" + case 0x02: + subType = "undelegations" + case 0x03: + subType = "undelegation_queue" + case 0x04: + subType = "receipts" + } + } + + if subType != "" { + return fmt.Sprintf("%s:%s", module, subType) + } + return module +} + // formatCBOR formats decoded CBOR value concisely func formatCBOR(v interface{}, rawLen int) string { switch val := v.(type) { From aa7131c20432a26723d00315296f3a801fd62b0f Mon Sep 17 00:00:00 2001 From: gw0 Date: Mon, 24 Nov 2025 11:47:49 +0100 Subject: [PATCH 11/22] badgerdb-sampler: Refactor decoding functionality and add EVM parsing --- badgerdb-sampler/decode_consensus.go | 622 ++++++++++++++++----------- badgerdb-sampler/decode_runtime.go | 454 ++++++++++++------- badgerdb-sampler/main.go | 64 +-- badgerdb-sampler/types_consensus.go | 152 +++++++ badgerdb-sampler/types_runtime.go | 195 +++++++++ badgerdb-sampler/utils.go | 9 +- 6 files changed, 1049 insertions(+), 447 deletions(-) create mode 100644 badgerdb-sampler/types_consensus.go create mode 100644 badgerdb-sampler/types_runtime.go diff --git a/badgerdb-sampler/decode_consensus.go b/badgerdb-sampler/decode_consensus.go index 2fcd262..f47fbf8 100644 --- a/badgerdb-sampler/decode_consensus.go +++ b/badgerdb-sampler/decode_consensus.go @@ -13,373 +13,377 @@ import ( tmstore "github.com/tendermint/tendermint/proto/tendermint/store" ) -// decodeKeyConsensusBlockstore parses consensus-blockstore key and returns key type and decoded representation -func decodeKeyConsensusBlockstore(key []byte) (string, string) { +// decodeKeyConsensusBlockstore parses consensus-blockstore key and returns structured info. +// See: tendermint/store/store.go for key formats (H:, P:, C:, SC:, BH:) +func decodeKeyConsensusBlockstore(key []byte) ConsensusBlockstoreKeyInfo { + info := ConsensusBlockstoreKeyInfo{} + if len(key) < 2 { - return "unknown", fmt.Sprintf("%x", key) + info.KeyType = "unknown" + return info } // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) if key[0] != 0x01 { - return "unknown", fmt.Sprintf("%x", key) + info.KeyType = "unknown" + return info } // Parse ASCII key after 0x01 prefix asciiKey := string(key[1:]) + info.RawKey = asciiKey // blockStore state key if asciiKey == "blockStore" { - return "blockstore_state", "blockStore" + info.KeyType = "blockstore_state" + return info } // Parse structured keys: "H:{height}", "P:{height}:{part}", "C:{height}", "SC:{height}", "BH:{hash}" if strings.HasPrefix(asciiKey, "H:") { - return "block_meta", asciiKey + info.KeyType = "block_meta" + if h, err := strconv.ParseInt(asciiKey[2:], 10, 64); err == nil { + info.Height = h + } + return info } if strings.HasPrefix(asciiKey, "P:") { - return "block_part", asciiKey + info.KeyType = "block_part" + // Format: P:{height}:{part} + parts := strings.Split(asciiKey[2:], ":") + if len(parts) >= 1 { + if h, err := strconv.ParseInt(parts[0], 10, 64); err == nil { + info.Height = h + } + } + if len(parts) >= 2 { + if p, err := strconv.Atoi(parts[1]); err == nil { + info.PartIndex = p + } + } + return info } if strings.HasPrefix(asciiKey, "C:") { - return "block_commit", asciiKey + info.KeyType = "block_commit" + if h, err := strconv.ParseInt(asciiKey[2:], 10, 64); err == nil { + info.Height = h + } + return info } if strings.HasPrefix(asciiKey, "SC:") { - return "seen_commit", asciiKey + info.KeyType = "seen_commit" + if h, err := strconv.ParseInt(asciiKey[3:], 10, 64); err == nil { + info.Height = h + } + return info } if strings.HasPrefix(asciiKey, "BH:") { - return "block_hash", asciiKey + info.KeyType = "block_hash" + info.Hash = asciiKey[3:] + return info } - return "unknown", asciiKey + info.KeyType = "unknown" + return info } -// decodeValueConsensusBlockstore attempts to decode protobuf value based on key type -// Returns: (decoded_string, timestamp_unix) -func decodeValueConsensusBlockstore(keyType string, value []byte) (string, int64) { +// decodeValueConsensusBlockstore decodes protobuf value and returns structured info. +// See: tendermint/proto/tendermint/store and tendermint/proto/tendermint/types +func decodeValueConsensusBlockstore(keyType string, value []byte) ConsensusBlockstoreValueInfo { + info := ConsensusBlockstoreValueInfo{ + KeyType: keyType, + Size: len(value), + } + switch keyType { case "blockstore_state": var state tmstore.BlockStoreState if err := proto.Unmarshal(value, &state); err == nil { - return fmt.Sprintf("BlockStoreState{base: %d, height: %d}", state.Base, state.Height), 0 + info.State = &ConsensusBlockStoreState{ + Base: state.Base, + Height: state.Height, + } + } else { + info.DecodeError = err.Error() } case "block_meta": var meta tmproto.BlockMeta if err := proto.Unmarshal(value, &meta); err == nil { + blockMeta := &ConsensusBlockMetaInfo{ + Height: meta.Header.Height, + NumTxs: meta.NumTxs, + } + // Extract timestamp - tsUnix := int64(0) - ts := "" if meta.Header.Time.Unix() > 0 { - tsUnix = meta.Header.Time.Unix() - ts = meta.Header.Time.Format(time.RFC3339) + info.Timestamp = meta.Header.Time.Unix() + blockMeta.Time = meta.Header.Time.Format(time.RFC3339) } - // Format AppHash and ChainID - appHash := "" + // Format AppHash if len(meta.Header.AppHash) >= 8 { - appHash = fmt.Sprintf("%x", meta.Header.AppHash[:8]) + blockMeta.AppHash = fmt.Sprintf("%x", meta.Header.AppHash[:8]) } else if len(meta.Header.AppHash) > 0 { - appHash = fmt.Sprintf("%x", meta.Header.AppHash) + blockMeta.AppHash = fmt.Sprintf("%x", meta.Header.AppHash) } + // Truncate ChainID chainID := meta.Header.ChainID if len(chainID) > 20 { chainID = chainID[:20] + "..." } + blockMeta.ChainID = chainID - return fmt.Sprintf("BlockMeta{height: %d, time: %s, chain: %s, num_txs: %d, app_hash: %s}", - meta.Header.Height, ts, chainID, meta.NumTxs, appHash), tsUnix + info.BlockMeta = blockMeta + } else { + info.DecodeError = err.Error() } case "block_part": var part tmproto.Part if err := proto.Unmarshal(value, &part); err == nil { - return fmt.Sprintf("Part{index: %d, bytes_size: %d, proof_total: %d}", - part.Index, len(part.Bytes), part.Proof.Total), 0 - } - - case "block_commit": - var commit tmproto.Commit - if err := proto.Unmarshal(value, &commit); err == nil { - return fmt.Sprintf("Commit{height: %d, round: %d, signatures: %d}", - commit.Height, commit.Round, len(commit.Signatures)), 0 + info.Part = &ConsensusPartInfo{ + Index: part.Index, + BytesSize: len(part.Bytes), + ProofTotal: part.Proof.Total, + } + } else { + info.DecodeError = err.Error() } - case "seen_commit": + case "block_commit", "seen_commit": var commit tmproto.Commit if err := proto.Unmarshal(value, &commit); err == nil { - return fmt.Sprintf("Commit{height: %d, round: %d, signatures: %d}", - commit.Height, commit.Round, len(commit.Signatures)), 0 + info.Commit = &ConsensusCommitInfo{ + Height: commit.Height, + Round: commit.Round, + Signatures: len(commit.Signatures), + } + } else { + info.DecodeError = err.Error() } case "block_hash": // Block hash values are plain strings containing height numbers - height, err := strconv.ParseInt(string(value), 10, 64) - if err == nil { - return fmt.Sprintf("Hash{height: %d}", height), 0 + if height, err := strconv.ParseInt(string(value), 10, 64); err == nil { + info.HashHeight = height + } else { + info.DecodeError = err.Error() } default: - return fmt.Sprintf("Unknown type (size: %d bytes)", len(value)), 0 + info.DecodeError = fmt.Sprintf("unknown key type: %s", keyType) } - return fmt.Sprintf("Failed to decode blockstore (type: %s, size: %d bytes)", keyType, len(value)), 0 + return info } -// decodeKeyConsensusEvidence parses consensus-evidence key and returns key type and decoded representation -func decodeKeyConsensusEvidence(key []byte) (string, string) { +// decodeKeyConsensusEvidence parses consensus-evidence key and returns structured info. +func decodeKeyConsensusEvidence(key []byte) ConsensusEvidenceKeyInfo { + info := ConsensusEvidenceKeyInfo{} + if len(key) < 2 { - return "unknown", fmt.Sprintf("%x", key) + info.KeyType = "unknown" + info.RawKey = fmt.Sprintf("%x", key) + return info } // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) if key[0] != 0x01 { - return "unknown", fmt.Sprintf("%x", key) + info.KeyType = "unknown" + info.RawKey = fmt.Sprintf("%x", key) + return info } // Evidence DB is typically empty or uses simple key patterns - return fmt.Sprintf("type_%02x", key[1]), fmt.Sprintf("%x", key[1:]) + info.KeyType = fmt.Sprintf("type_%02x", key[1]) + info.PrefixByte = key[1] + info.RawKey = fmt.Sprintf("%x", key[1:]) + return info } -// decodeValueConsensusEvidence attempts to decode evidence value -func decodeValueConsensusEvidence(keyType string, value []byte) string { +// decodeValueConsensusEvidence decodes evidence value and returns structured info. +func decodeValueConsensusEvidence(keyType string, value []byte) ConsensusEvidenceValueInfo { + info := ConsensusEvidenceValueInfo{ + Size: len(value), + } + if len(value) == 0 { - return "Empty" + return info } // Try to decode as DuplicateVoteEvidence var evidence tmproto.DuplicateVoteEvidence if err := proto.Unmarshal(value, &evidence); err == nil { - return fmt.Sprintf("DuplicateVoteEvidence{vote_a_height: %d, vote_b_height: %d}", - evidence.VoteA.Height, evidence.VoteB.Height) + if evidence.VoteA != nil { + info.VoteAHeight = evidence.VoteA.Height + } + if evidence.VoteB != nil { + info.VoteBHeight = evidence.VoteB.Height + } + } else { + info.DecodeError = err.Error() } - return fmt.Sprintf("Evidence (type: %s, size: %d bytes)", keyType, len(value)) + return info } -// decodeKeyConsensusMkvs parses consensus-mkvs key and returns key type and decoded representation -// FIXED: Handles both old format (no dbVersion prefix) and new format (0x01 or 0x05 dbVersion prefix) -// MKVS keys from oasis-core/go/storage/mkvs/db/badger/badger.go -func decodeKeyConsensusMkvs(key []byte) (string, string) { +// decodeKeyConsensusMkvs parses consensus-mkvs key and returns structured info. +// Handles both old format (no dbVersion prefix) and new format (0x01 or 0x05 dbVersion prefix). +// See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 +func decodeKeyConsensusMkvs(key []byte) ConsensusMkvsKeyInfo { + info := ConsensusMkvsKeyInfo{} + if len(key) < 1 { - return "unknown", fmt.Sprintf("%x", key) + info.KeyType = "unknown" + info.DecodeError = "key too short" + return info } // Check if key has dbVersion prefix (0x01 or 0x05) - // Old format: type byte directly at key[0] - // New format: 0x01 or 0x05 at key[0], type byte at key[1] prefixByte := key[0] var data []byte if prefixByte == 0x01 || prefixByte == 0x05 { - // New format with dbVersion prefix + info.DbPrefix = prefixByte if len(key) < 2 { - return "unknown", fmt.Sprintf("%x", key) + info.KeyType = "unknown" + info.DecodeError = "key too short after prefix" + return info } prefixByte = key[1] data = key[2:] } else { - // Old format without dbVersion prefix (BadgerDB v2 MKVS) data = key[1:] } switch prefixByte { case 0x00: - // nodeKeyFmt: hash.Hash (32 bytes) - return "node", fmt.Sprintf("node{hash: %x....}", truncateBytes(data, 8)) + info.KeyType = "node" + info.Hash = truncateHex(data, 16) + case 0x01: - // writeLogKeyFmt: uint64(version) + TypedHash(new root) + TypedHash(old root) + info.KeyType = "write_log" if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - if len(data) >= 8+33 { // version + first TypedHash (33 bytes in v3+) - rootType := data[8] - rootHash := data[9 : 9+32] - return "write_log", fmt.Sprintf("write_log{v:%d, new_root_type:%d, hash: %x...}", version, rootType, truncateBytes(rootHash, 4)) + info.Height = binary.BigEndian.Uint64(data[0:8]) + if len(data) >= 8+33 { + info.RootType = data[8] + info.Hash = truncateHex(data[9:9+32], 8) } - return "write_log", fmt.Sprintf("write_log{v:%d}", version) + } else { + info.DecodeError = "write_log data too short" } - return "write_log", "write_log{}" + case 0x02: - // rootsMetadataKeyFmt: uint64(version) + info.KeyType = "roots_metadata" if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - return "roots_metadata", fmt.Sprintf("roots_metadata{v:%d}", version) + info.Height = binary.BigEndian.Uint64(data[0:8]) + } else { + info.DecodeError = "roots_metadata data too short" } - return "roots_metadata", "roots_metadata{}" + case 0x03: - // rootUpdatedNodesKeyFmt: uint64(version) + TypedHash(root) + info.KeyType = "root_updated_nodes" if len(data) >= 8 { - version := binary.BigEndian.Uint64(data[0:8]) - if len(data) >= 8+33 { // version + TypedHash - rootType := data[8] - rootHash := data[9 : 9+32] - return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{v:%d, root_type:%d, hash:%x...}", version, rootType, truncateBytes(rootHash, 4)) + info.Height = binary.BigEndian.Uint64(data[0:8]) + if len(data) >= 8+33 { + info.RootType = data[8] + info.Hash = truncateHex(data[9:9+32], 8) } - return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{v:%d}", version) + } else { + info.DecodeError = "root_updated_nodes data too short" } - return "root_updated_nodes", "root_updated_nodes{}" + case 0x04: - // metadataKeyFmt: no additional data - return "metadata", "metadata" + info.KeyType = "metadata" + case 0x05: - // multipartRestoreNodeLogKeyFmt: TypedHash (33 bytes) + info.KeyType = "multipart_restore_log" if len(data) >= 33 { - rootType := data[0] - rootHash := data[1:33] - return "multipart_restore_log", fmt.Sprintf("multipart_restore_log{type:%d, hash:%x}", rootType, truncateBytes(rootHash, 8)) + info.RootType = data[0] + info.Hash = truncateHex(data[1:33], 16) + } else { + info.Hash = truncateHex(data, 16) } - return "multipart_restore_log", fmt.Sprintf("multipart_restore_log{hash:%x}", truncateBytes(data, 8)) + case 0x06: - // rootNodeKeyFmt: TypedHash (33 bytes) + info.KeyType = "root_node" if len(data) >= 33 { - rootType := data[0] - rootHash := data[1:33] - return "root_node", fmt.Sprintf("root_node{type:%d, hash:%x}", rootType, truncateBytes(rootHash, 8)) + info.RootType = data[0] + info.Hash = truncateHex(data[1:33], 16) + } else { + info.Hash = truncateHex(data, 16) } - return "root_node", fmt.Sprintf("root_node{hash:%x}", truncateBytes(data, 8)) + default: - return fmt.Sprintf("unknown_%02x", prefixByte), fmt.Sprintf("unknown_%02x:%x", prefixByte, truncateBytes(data, 8)) + info.KeyType = fmt.Sprintf("unknown_%02x", prefixByte) + info.DecodeError = fmt.Sprintf("unknown key prefix 0x%02x", prefixByte) } + + return info } -// decodeValueConsensusMkvs decodes consensus MKVS value with inline node parsing -func decodeValueConsensusMkvs(keyType string, value []byte) string { +// decodeValueConsensusMkvs decodes consensus MKVS value and returns structured info. +// See: _oasis-core/go/storage/mkvs/node/node.go:26-32 (prefixes), 294-309 (InternalNode), 531-537 (LeafNode) +func decodeValueConsensusMkvs(keyType string, value []byte) ConsensusMkvsNodeInfo { + info := ConsensusMkvsNodeInfo{Size: len(value)} + if len(value) == 0 { - return "Empty" + info.NodeType = "empty" + return info } if keyType != "node" { - return fmt.Sprintf("MKVS %s (size: %d bytes)", keyType, len(value)) + info.NodeType = "non_node" + return info } // Parse MKVS node: 0x00=leaf, 0x01=internal, 0x02=nil switch value[0] { - case 0x00: // LeafNode: [2-byte keyLen LE][key][4-byte valueLen LE][value] + case 0x00: // LeafNode + info.NodeType = "leaf" data := value[1:] + if len(data) < 2 { - return "LeafNode{}" + info.DecodeError = "key length missing" + return info } keyLen := int(binary.LittleEndian.Uint16(data[0:2])) data = data[2:] + if len(data) < keyLen { - return fmt.Sprintf("LeafNode{keyLen=%d, }", keyLen) + info.DecodeError = fmt.Sprintf("key truncated (expected %d bytes)", keyLen) + return info } key := data[:keyLen] data = data[keyLen:] - // Decode consensus module key prefix (roothash, staking, registry, etc.) - // Consensus modules use single-byte prefixes from oasis-core/go/consensus/tendermint/apps/*/state/state.go - var module string - if len(key) == 0 { - module = "" - } else { - switch key[0] { - // Roothash module (0x20-0x29) - case 0x20: - module = "roothash/runtime_state" - case 0x21: - module = "roothash/params" - case 0x22: - module = "roothash/round_timeout" - case 0x24: - module = "roothash/evidence" - case 0x25: - module = "roothash/state_root" - case 0x26: - module = "roothash/io_root" - case 0x27: - module = "roothash/last_round_results" - case 0x28: - module = "roothash/incoming_msg_queue_meta" - case 0x29: - module = "roothash/incoming_msg_queue" - // Staking module (0x30-0x3F) - case 0x30: - module = "staking/total_supply" - case 0x31: - module = "staking/common_pool" - case 0x32: - module = "staking/last_block_fees" - case 0x33: - module = "staking/governance_deposits" - case 0x34: - module = "staking/accounts" - case 0x35: - module = "staking/delegations" - case 0x36: - module = "staking/debonding_delegations" - case 0x37: - module = "staking/allowances" - case 0x38: - module = "staking/params" - // Registry module (0x40-0x4F) - case 0x40: - module = "registry/entities" - case 0x41: - module = "registry/nodes" - case 0x42: - module = "registry/node_by_consensus" - case 0x43: - module = "registry/runtimes" - case 0x44: - module = "registry/suspended_runtimes" - case 0x45: - module = "registry/params" - case 0x46: - module = "registry/node_status" - // Scheduler module (0x50-0x5F) - case 0x50: - module = "scheduler/params" - case 0x51: - module = "scheduler/committees" - case 0x52: - module = "scheduler/validators" - // Governance module (0x60-0x6F) - case 0x60: - module = "governance/params" - case 0x61: - module = "governance/proposals" - case 0x62: - module = "governance/active_proposals" - case 0x63: - module = "governance/votes" - case 0x64: - module = "governance/pending_upgrades" - // Beacon module (0x70-0x7F) - case 0x70: - module = "beacon/params" - case 0x71: - module = "beacon/future_epoch" - case 0x72: - module = "beacon/epoch" - case 0x73: - module = "beacon/pvss_state" - // Keymanager module (0x80-0x8F) - case 0x80: - module = "keymanager/status" - case 0x81: - module = "keymanager/params" - // Consensus parameters - case 0xF1: - module = "consensus/params" - default: - if key[0] >= 'a' && key[0] <= 'z' { - module = extractModuleName(key) - } else { - module = fmt.Sprintf("0x%02x", key[0]) - } - } + // Decode consensus module key prefix + module := decodeConsensusModulePrefix(key) + + leaf := &ConsensusMkvsLeafInfo{ + Module: module, + KeyLen: keyLen, + Key: truncateHex(key, 32), } if len(data) < 4 { - return fmt.Sprintf("LeafNode{module=%s, }", module) + info.DecodeError = "value length missing" + info.Leaf = leaf + return info } valueLen := int(binary.LittleEndian.Uint32(data[0:4])) data = data[4:] + leaf.ValueLen = valueLen + if len(data) < valueLen { - return fmt.Sprintf("LeafNode{module=%s, valueLen=%d, }", module, valueLen) + info.DecodeError = fmt.Sprintf("value truncated (expected %d bytes)", valueLen) + info.Leaf = leaf + return info } leafValue := data[:valueLen] @@ -387,15 +391,21 @@ func decodeValueConsensusMkvs(keyType string, value []byte) string { // Try CBOR decode var decoded interface{} if err := cbor.Unmarshal(leafValue, &decoded); err == nil { - return fmt.Sprintf("LeafNode{module=%s, value=%s}", module, formatCBOR(decoded, valueLen)) + leaf.DecodedValue = formatCBOR(decoded, valueLen) + } else { + leaf.DecodedValue = fmt.Sprintf("binary(%d bytes)", valueLen) } - return fmt.Sprintf("LeafNode{module=%s, value=binary(%d bytes)}", module, valueLen) + info.Leaf = leaf + return info - case 0x01: // InternalNode: [2-byte labelBits LE][label][leaf/nil marker][hashes] + case 0x01: // InternalNode + info.NodeType = "internal" data := value[1:] + if len(data) < 2 { - return "InternalNode{}" + info.DecodeError = "label bits missing" + return info } labelBits := binary.LittleEndian.Uint16(data[0:2]) @@ -403,47 +413,159 @@ func decodeValueConsensusMkvs(keyType string, value []byte) string { labelBytes := (int(labelBits) + 7) / 8 if len(data) < labelBytes+1 { - return fmt.Sprintf("InternalNode{label=%d bits, }", labelBits) + info.DecodeError = "label truncated" + info.Internal = &ConsensusMkvsInternalInfo{LabelBits: labelBits} + return info } data = data[labelBytes:] // skip label + internal := &ConsensusMkvsInternalInfo{LabelBits: labelBits} hasLeaf := data[0] == 0x00 - if hasLeaf { - return fmt.Sprintf("InternalNode{label=%d bits, has_leaf=true}", labelBits) - } + internal.HasLeaf = hasLeaf - data = data[1:] // skip nil marker - - // Extract child hashes - var left, right string - if len(data) >= 32 { - left = truncateHex(data[:32], 16) - data = data[32:] - } - if len(data) >= 32 { - right = truncateHex(data[:32], 16) + if !hasLeaf { + data = data[1:] // skip nil marker + if len(data) >= 32 { + internal.LeftHash = truncateHex(data[:32], 16) + data = data[32:] + } + if len(data) >= 32 { + internal.RightHash = truncateHex(data[:32], 16) + } } - return fmt.Sprintf("InternalNode{label=%d bits, left=%s, right=%s}", labelBits, left, right) + info.Internal = internal + return info case 0x02: // NilNode - return "NilNode{}" + info.NodeType = "nil" + return info default: - return fmt.Sprintf("Unknown node prefix 0x%02x (size: %d)", value[0], len(value)) + info.NodeType = "unknown" + info.DecodeError = fmt.Sprintf("unknown prefix 0x%02x", value[0]) + return info } } -// decodeKeyConsensusState parses consensus-state key and returns key type and decoded representation -func decodeKeyConsensusState(key []byte) (string, string) { +// decodeConsensusModulePrefix decodes the consensus module key prefix. +// See: _oasis-core/go/consensus/tendermint/apps/*/state/state.go +func decodeConsensusModulePrefix(key []byte) string { + if len(key) == 0 { + return "" + } + + switch key[0] { + // Roothash module (0x20-0x29) + case 0x20: + return "roothash/runtime_state" + case 0x21: + return "roothash/params" + case 0x22: + return "roothash/round_timeout" + case 0x24: + return "roothash/evidence" + case 0x25: + return "roothash/state_root" + case 0x26: + return "roothash/io_root" + case 0x27: + return "roothash/last_round_results" + case 0x28: + return "roothash/incoming_msg_queue_meta" + case 0x29: + return "roothash/incoming_msg_queue" + // Staking module (0x30-0x3F) + case 0x30: + return "staking/total_supply" + case 0x31: + return "staking/common_pool" + case 0x32: + return "staking/last_block_fees" + case 0x33: + return "staking/governance_deposits" + case 0x34: + return "staking/accounts" + case 0x35: + return "staking/delegations" + case 0x36: + return "staking/debonding_delegations" + case 0x37: + return "staking/allowances" + case 0x38: + return "staking/params" + // Registry module (0x40-0x4F) + case 0x40: + return "registry/entities" + case 0x41: + return "registry/nodes" + case 0x42: + return "registry/node_by_consensus" + case 0x43: + return "registry/runtimes" + case 0x44: + return "registry/suspended_runtimes" + case 0x45: + return "registry/params" + case 0x46: + return "registry/node_status" + // Scheduler module (0x50-0x5F) + case 0x50: + return "scheduler/params" + case 0x51: + return "scheduler/committees" + case 0x52: + return "scheduler/validators" + // Governance module (0x60-0x6F) + case 0x60: + return "governance/params" + case 0x61: + return "governance/proposals" + case 0x62: + return "governance/active_proposals" + case 0x63: + return "governance/votes" + case 0x64: + return "governance/pending_upgrades" + // Beacon module (0x70-0x7F) + case 0x70: + return "beacon/params" + case 0x71: + return "beacon/future_epoch" + case 0x72: + return "beacon/epoch" + case 0x73: + return "beacon/pvss_state" + // Keymanager module (0x80-0x8F) + case 0x80: + return "keymanager/status" + case 0x81: + return "keymanager/params" + // Consensus parameters + case 0xF1: + return "consensus/params" + default: + if key[0] >= 'a' && key[0] <= 'z' { + return extractModuleName(key) + } + return fmt.Sprintf("0x%02x", key[0]) + } +} + +// decodeKeyConsensusState parses consensus-state key and returns structured info. +func decodeKeyConsensusState(key []byte) ConsensusStateKeyInfo { + info := ConsensusStateKeyInfo{} + if len(key) < 2 { - return "unknown", fmt.Sprintf("%x", key) + info.KeyType = "unknown" + return info } // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) if key[0] != 0x01 { - return "unknown", fmt.Sprintf("%x", key) + info.KeyType = "unknown" + return info } // Parse ASCII key after 0x01 prefix @@ -451,36 +573,42 @@ func decodeKeyConsensusState(key []byte) (string, string) { // Check if it's printable ASCII if !isPrintableASCII(decoded) { - return "binary", fmt.Sprintf("%x", key) + info.KeyType = "binary" + info.IsBinary = true + return info } + info.DecodedKey = decoded + // Extract key type prefix before colon colonIdx := strings.IndexByte(decoded, ':') if colonIdx != -1 { prefix := decoded[:colonIdx] switch prefix { case "abciResponsesKey": - return "abci_responses", decoded + info.KeyType = "abci_responses" case "consensusParamsKey": - return "consensus_params", decoded + info.KeyType = "consensus_params" case "validatorsKey": - return "validators", decoded + info.KeyType = "validators" case "stateKey": - return "state", decoded + info.KeyType = "state" case "genesisDoc": - return "genesis", decoded + info.KeyType = "genesis" default: - return prefix, decoded + info.KeyType = prefix } + return info } - return "text_key", decoded + info.KeyType = "text_key" + return info } -// decodeValueConsensusState decodes state value -func decodeValueConsensusState(keyType string, value []byte) string { - if len(value) == 0 { - return "Empty" +// decodeValueConsensusState decodes state value and returns structured info. +func decodeValueConsensusState(keyType string, value []byte) ConsensusStateValueInfo { + return ConsensusStateValueInfo{ + KeyType: keyType, + Size: len(value), } - return fmt.Sprintf("Tendermint %s data (size: %d bytes)", keyType, len(value)) } diff --git a/badgerdb-sampler/decode_runtime.go b/badgerdb-sampler/decode_runtime.go index e69d6f8..f19880b 100644 --- a/badgerdb-sampler/decode_runtime.go +++ b/badgerdb-sampler/decode_runtime.go @@ -8,76 +8,16 @@ import ( "github.com/fxamacker/cbor/v2" ) -// Runtime history CBOR types (from oasis-core/go/runtime/history/db.go) - -// RuntimeHistoryMetadata represents the metadata stored in runtime history DB -type RuntimeHistoryMetadata struct { - RuntimeID []byte `cbor:"runtime_id"` - Version uint64 `cbor:"version"` - LastConsensusHeight int64 `cbor:"last_consensus_height"` - LastRound uint64 `cbor:"last_round"` -} - -// RuntimeHistoryAnnotatedBlock represents an annotated block in runtime history -type RuntimeHistoryAnnotatedBlock struct { - Height int64 `cbor:"consensus_height"` - Block *RuntimeHistoryBlock `cbor:"block"` -} - -// RuntimeHistoryBlock represents a runtime block -type RuntimeHistoryBlock struct { - Header RuntimeHistoryBlockHeader `cbor:"header"` -} - -// RuntimeHistoryBlockHeader represents a runtime block header -type RuntimeHistoryBlockHeader struct { - Version uint16 `cbor:"version"` - Namespace []byte `cbor:"namespace"` - Round uint64 `cbor:"round"` - Timestamp uint64 `cbor:"timestamp"` - HeaderType uint8 `cbor:"header_type"` - PreviousHash []byte `cbor:"previous_hash"` - IORoot []byte `cbor:"io_root"` - StateRoot []byte `cbor:"state_root"` - MessagesHash []byte `cbor:"messages_hash"` - InMessagesHash []byte `cbor:"in_msgs_hash"` -} - -// RuntimeHistoryRoundResults represents round results in runtime history -type RuntimeHistoryRoundResults struct { - Messages []RuntimeHistoryMessageEvent `cbor:"messages,omitempty"` - GoodComputeEntities [][]byte `cbor:"good_compute_entities,omitempty"` - BadComputeEntities [][]byte `cbor:"bad_compute_entities,omitempty"` -} +// decodeKeyRuntimeMkvs parses runtime-mkvs key and returns structured info. +// Key format: [optional db prefix 0x01|0x05][type byte][data...] +// See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 +func decodeKeyRuntimeMkvs(key []byte) RuntimeMkvsKeyInfo { + info := RuntimeMkvsKeyInfo{} -// RuntimeHistoryMessageEvent represents a message event -type RuntimeHistoryMessageEvent struct { - Module string `cbor:"module,omitempty"` - Code uint32 `cbor:"code,omitempty"` - Index uint32 `cbor:"index,omitempty"` - Result cbor.RawMessage `cbor:"result,omitempty"` -} - -// RuntimeInputArtifacts represents input transaction artifacts stored in IO tree -// From oasis-core/go/runtime/transaction/transaction.go -type RuntimeInputArtifacts struct { - _ struct{} `cbor:",toarray"` - Input []byte - BatchOrder uint32 -} - -// RuntimeOutputArtifacts represents output transaction artifacts stored in IO tree -// From oasis-core/go/runtime/transaction/transaction.go -type RuntimeOutputArtifacts struct { - _ struct{} `cbor:",toarray"` - Output []byte -} - -// decodeKeyRuntimeMkvs parses runtime-mkvs key -// Key format: [prefix byte][type byte][data...] -func decodeKeyRuntimeMkvs(key []byte) (string, string) { if len(key) < 1 { - return "unknown", fmt.Sprintf("%x", key) + info.KeyType = "unknown" + info.DecodeError = "key too short" + return info } // Check for dbVersion prefix (0x01 or 0x05) @@ -85,8 +25,11 @@ func decodeKeyRuntimeMkvs(key []byte) (string, string) { var data []byte if prefixByte == 0x01 || prefixByte == 0x05 { + info.DbPrefix = prefixByte if len(key) < 2 { - return "unknown", fmt.Sprintf("%x", key) + info.KeyType = "unknown" + info.DecodeError = "key too short after prefix" + return info } prefixByte = key[1] data = key[2:] @@ -96,82 +39,113 @@ func decodeKeyRuntimeMkvs(key []byte) (string, string) { switch prefixByte { case 0x00: - return "node", fmt.Sprintf("node{hash: %s}", truncateHex(data, 16)) + info.KeyType = "node" + info.Hash = truncateHex(data, 16) case 0x01: + info.KeyType = "write_log" if len(data) >= 8 { - height := binary.BigEndian.Uint64(data[0:8]) - return "write_log", fmt.Sprintf("write_log{height:%d}", height) + info.Height = binary.BigEndian.Uint64(data[0:8]) + } else { + info.DecodeError = "write_log data too short" } - return "write_log", "write_log{}" case 0x02: + info.KeyType = "roots_metadata" if len(data) >= 8 { - height := binary.BigEndian.Uint64(data[0:8]) - return "roots_metadata", fmt.Sprintf("roots_metadata{height:%d}", height) + info.Height = binary.BigEndian.Uint64(data[0:8]) + } else { + info.DecodeError = "roots_metadata data too short" } - return "roots_metadata", "roots_metadata{}" case 0x03: + info.KeyType = "root_updated_nodes" if len(data) >= 8 { - height := binary.BigEndian.Uint64(data[0:8]) - return "root_updated_nodes", fmt.Sprintf("root_updated_nodes{height:%d}", height) + info.Height = binary.BigEndian.Uint64(data[0:8]) + } else { + info.DecodeError = "root_updated_nodes data too short" } - return "root_updated_nodes", "root_updated_nodes{}" case 0x04: - return "metadata", "metadata" + info.KeyType = "metadata" default: - return fmt.Sprintf("unknown_%02x", prefixByte), fmt.Sprintf("%x", key) + info.KeyType = fmt.Sprintf("unknown_%02x", prefixByte) + info.DecodeError = fmt.Sprintf("unknown key prefix 0x%02x", prefixByte) } + + return info } -// decodeValueRuntimeMkvs decodes runtime MKVS value with inline node parsing -func decodeValueRuntimeMkvs(keyType string, value []byte) string { +// decodeValueRuntimeMkvs decodes runtime MKVS value and returns structured info. +// See: _oasis-core/go/storage/mkvs/node/node.go:26-32 (prefixes), 294-309 (InternalNode), 531-537 (LeafNode) +func decodeValueRuntimeMkvs(keyType string, value []byte) RuntimeMkvsNodeInfo { + info := RuntimeMkvsNodeInfo{Size: len(value)} + if len(value) == 0 { - return "Empty" + info.NodeType = "empty" + return info } if keyType != "node" { - return fmt.Sprintf("MKVS %s (size: %d bytes)", keyType, len(value)) + info.NodeType = "non_node" + return info } // Parse MKVS node: 0x00=leaf, 0x01=internal, 0x02=nil switch value[0] { case 0x00: // LeafNode: [2-byte keyLen LE][key][4-byte valueLen LE][value] + info.NodeType = "leaf" data := value[1:] + if len(data) < 2 { - return "LeafNode{}" + info.DecodeError = "key length missing" + return info } keyLen := int(binary.LittleEndian.Uint16(data[0:2])) data = data[2:] + if len(data) < keyLen { - return fmt.Sprintf("LeafNode{keyLen=%d, }", keyLen) + info.DecodeError = fmt.Sprintf("key truncated (expected %d bytes)", keyLen) + return info } key := data[:keyLen] data = data[keyLen:] - // Extract module name from key (runtime-specific: evm, contracts, accounts, etc.) + // Extract module name from key module := extractModuleName(key) + leaf := &RuntimeMkvsLeafInfo{ + Module: module, + KeyLen: keyLen, + Key: truncateHex(key, 32), + } + if len(data) < 4 { - return fmt.Sprintf("LeafNode{module=%s, }", module) + info.DecodeError = "value length missing" + info.Leaf = leaf + return info } valueLen := int(binary.LittleEndian.Uint32(data[0:4])) data = data[4:] + leaf.ValueLen = valueLen + if len(data) < valueLen { - return fmt.Sprintf("LeafNode{module=%s, valueLen=%d, }", module, valueLen) + info.DecodeError = fmt.Sprintf("value truncated (expected %d bytes)", valueLen) + info.Leaf = leaf + return info } leafValue := data[:valueLen] - - // Try to decode based on module type - valueDesc := decodeLeafValue(module, key, leafValue) - return fmt.Sprintf("LeafNode{module=%s, value=%s}", module, valueDesc) + leaf.DecodedValue = decodeLeafValue(module, key, leafValue) + info.Leaf = leaf + return info case 0x01: // InternalNode: [2-byte labelBits LE][label][leaf/nil marker][hashes] + info.NodeType = "internal" data := value[1:] + if len(data) < 2 { - return "InternalNode{}" + info.DecodeError = "label bits missing" + return info } labelBits := binary.LittleEndian.Uint16(data[0:2]) @@ -179,121 +153,155 @@ func decodeValueRuntimeMkvs(keyType string, value []byte) string { labelBytes := (int(labelBits) + 7) / 8 if len(data) < labelBytes+1 { - return fmt.Sprintf("InternalNode{label=%d bits, }", labelBits) + info.DecodeError = "label truncated" + info.Internal = &RuntimeMkvsInternalInfo{LabelBits: labelBits} + return info } data = data[labelBytes:] // skip label + internal := &RuntimeMkvsInternalInfo{LabelBits: labelBits} hasLeaf := data[0] == 0x00 - if hasLeaf { - return fmt.Sprintf("InternalNode{label=%d bits, has_leaf=true}", labelBits) - } - - data = data[1:] // skip nil marker - - // Extract child hashes - var left, right string - if len(data) >= 32 { - left = truncateHex(data[:32], 16) - data = data[32:] - } - if len(data) >= 32 { - right = truncateHex(data[:32], 16) + internal.HasLeaf = hasLeaf + + if !hasLeaf { + data = data[1:] // skip nil marker + // Extract child hashes + if len(data) >= 32 { + internal.LeftHash = truncateHex(data[:32], 16) + data = data[32:] + } + if len(data) >= 32 { + internal.RightHash = truncateHex(data[:32], 16) + } } - return fmt.Sprintf("InternalNode{label=%d bits, left=%s, right=%s}", labelBits, left, right) + info.Internal = internal + return info case 0x02: // NilNode - return "NilNode{}" + info.NodeType = "nil" + return info default: - return fmt.Sprintf("Unknown node prefix 0x%02x (size: %d)", value[0], len(value)) + info.NodeType = "unknown" + info.DecodeError = fmt.Sprintf("unknown prefix 0x%02x", value[0]) + return info } } -// decodeKeyRuntimeHistory parses runtime-history key using binary prefix format -// Key formats from oasis-core/go/runtime/history/db.go: +// decodeKeyRuntimeHistory parses runtime-history key and returns structured info. +// See: _oasis-core/go/runtime/history/db.go:19-31 +// Key formats: // - 0x01: metadata key (1 byte) // - 0x02 + uint64: block key (9 bytes) - round number in big-endian // - 0x03 + uint64: round_results key (9 bytes) - round number in big-endian -func decodeKeyRuntimeHistory(key []byte) (string, string) { +func decodeKeyRuntimeHistory(key []byte) RuntimeHistoryKeyInfo { + info := RuntimeHistoryKeyInfo{} + if len(key) == 0 { - return "unknown", "" + info.KeyType = "unknown" + info.DecodeError = "empty key" + return info } switch key[0] { case 0x01: - // Metadata key - just the prefix byte - if len(key) == 1 { - return "metadata", "metadata" + info.KeyType = "metadata" + if len(key) > 1 { + info.ExtraData = fmt.Sprintf("%x", key[1:]) } - // Unexpected extra data after metadata prefix - return "metadata", fmt.Sprintf("metadata (extra: %x)", key[1:]) case 0x02: - // Block key - prefix + 8-byte round number + info.KeyType = "block" if len(key) == 9 { - round := binary.BigEndian.Uint64(key[1:9]) - return "block", fmt.Sprintf("round:%d", round) + info.Height = binary.BigEndian.Uint64(key[1:9]) + } else { + info.DecodeError = "block key wrong length" } - return "block", fmt.Sprintf("block (malformed, len=%d)", len(key)) case 0x03: - // Round results key - prefix + 8-byte round number + info.KeyType = "round_results" if len(key) == 9 { - round := binary.BigEndian.Uint64(key[1:9]) - return "round_results", fmt.Sprintf("round:%d", round) + info.Height = binary.BigEndian.Uint64(key[1:9]) + } else { + info.DecodeError = "round_results key wrong length" } - return "round_results", fmt.Sprintf("round_results (malformed, len=%d)", len(key)) default: - return "unknown", fmt.Sprintf("%x", key) + info.KeyType = "unknown" + info.DecodeError = fmt.Sprintf("unknown key prefix 0x%02x", key[0]) } + + return info } -// decodeValueRuntimeHistory decodes CBOR-encoded runtime history value -func decodeValueRuntimeHistory(keyType string, value []byte) string { +// decodeValueRuntimeHistory decodes CBOR-encoded runtime history value and returns structured info. +// See: _oasis-core/go/runtime/history/db.go:34-44 (metadata) +// See: _oasis-core/go/roothash/api/api.go:402-409 (AnnotatedBlock) +// See: _oasis-core/go/roothash/api/results.go:5-17 (RoundResults) +func decodeValueRuntimeHistory(keyType string, value []byte) RuntimeHistoryValueInfo { + info := RuntimeHistoryValueInfo{ + KeyType: keyType, + Size: len(value), + } + if len(value) == 0 { - return "Empty" + info.DecodeError = "empty value" + return info } switch keyType { case "metadata": var meta RuntimeHistoryMetadata if err := cbor.Unmarshal(value, &meta); err != nil { - return fmt.Sprintf("Failed to decode metadata: %v (size: %d)", err, len(value)) + info.DecodeError = err.Error() + return info + } + info.Metadata = &RuntimeHistoryMetadataInfo{ + Version: meta.Version, + RuntimeID: truncateHex(meta.RuntimeID, 16), + LastRound: meta.LastRound, + LastConsensusHeight: meta.LastConsensusHeight, } - runtimeID := truncateHex(meta.RuntimeID, 16) - return fmt.Sprintf("Metadata{version: %d, runtime_id: %s..., last_round: %d, last_consensus_height: %d}", - meta.Version, runtimeID, meta.LastRound, meta.LastConsensusHeight) case "block": var block RuntimeHistoryAnnotatedBlock if err := cbor.Unmarshal(value, &block); err != nil { - return fmt.Sprintf("Failed to decode block: %v (size: %d)", err, len(value)) + info.DecodeError = err.Error() + return info + } + blockInfo := &RuntimeHistoryBlockInfo{ + ConsensusHeight: block.Height, } if block.Block == nil { - return fmt.Sprintf("AnnotatedBlock{consensus_height: %d, block: nil}", block.Height) + blockInfo.BlockNil = true + } else { + h := block.Block.Header + blockInfo.Round = h.Round + blockInfo.Timestamp = time.Unix(int64(h.Timestamp), 0).UTC().Format(time.RFC3339) + blockInfo.HeaderType = headerTypeName(h.HeaderType) + blockInfo.StateRoot = truncateHex(h.StateRoot, 16) } - h := block.Block.Header - // Format timestamp as human-readable - ts := time.Unix(int64(h.Timestamp), 0).UTC().Format(time.RFC3339) - headerType := headerTypeName(h.HeaderType) - stateRoot := truncateHex(h.StateRoot, 16) - return fmt.Sprintf("AnnotatedBlock{consensus_height: %d, round: %d, timestamp: %s, header_type: %s, state_root: %s...}", - block.Height, h.Round, ts, headerType, stateRoot) + info.Block = blockInfo case "round_results": var results RuntimeHistoryRoundResults if err := cbor.Unmarshal(value, &results); err != nil { - return fmt.Sprintf("Failed to decode round_results: %v (size: %d)", err, len(value)) + info.DecodeError = err.Error() + return info + } + info.RoundResults = &RuntimeHistoryRoundResultsInfo{ + MessageCount: len(results.Messages), + GoodComputeEntities: len(results.GoodComputeEntities), + BadComputeEntities: len(results.BadComputeEntities), } - return fmt.Sprintf("RoundResults{messages: %d, good_entities: %d, bad_entities: %d}", - len(results.Messages), len(results.GoodComputeEntities), len(results.BadComputeEntities)) default: - return fmt.Sprintf("Runtime %s data (size: %d bytes)", keyType, len(value)) + info.DecodeError = fmt.Sprintf("unknown key type: %s", keyType) } + + return info } // headerTypeName converts header type byte to string @@ -314,8 +322,31 @@ func headerTypeName(headerType uint8) string { } } -// decodeLeafValue decodes MKVS leaf value based on module type -func decodeLeafValue(module string, key []byte, value []byte) string { +// decodeLeafValue decodes MKVS leaf value and returns structured info. +// See: _oasis-core/go/runtime/transaction/transaction.go:129-150 (artifacts) +func decodeLeafValue(module string, key []byte, value []byte) *RuntimeLeafValueInfo { + info := &RuntimeLeafValueInfo{} + + // Check for EVM module data + if len(module) >= 3 && module[:3] == "evm" { + evmInfo := decodeEVMData(module, key, value) + if evmInfo != nil { + info.EVM = evmInfo + // Set value_type based on EVM storage type + switch evmInfo.StorageType { + case "code": + info.ValueType = "evm_code" + case "storage", "confidential_storage": + info.ValueType = "evm_storage" + case "block_hash": + info.ValueType = "evm_block_hash" + default: + info.ValueType = "evm_unknown" + } + return info + } + } + // Check for IO transaction artifacts if len(module) > 5 && module[:5] == "io_tx" { // Determine artifact kind from key @@ -325,13 +356,18 @@ func decodeLeafValue(module string, key []byte, value []byte) string { // Input artifact var ia RuntimeInputArtifacts if err := cbor.Unmarshal(value, &ia); err == nil { - return fmt.Sprintf("RuntimeInputArtifacts{input_size=%d, batch_order=%d}", len(ia.Input), ia.BatchOrder) + info.ValueType = "io_input" + info.InputSize = len(ia.Input) + info.BatchOrder = ia.BatchOrder + return info } } else if kind == 2 { // Output artifact var oa RuntimeOutputArtifacts if err := cbor.Unmarshal(value, &oa); err == nil { - return fmt.Sprintf("RuntimeOutputArtifacts{output_size=%d}", len(oa.Output)) + info.ValueType = "io_output" + info.OutputSize = len(oa.Output) + return info } } } @@ -339,19 +375,117 @@ func decodeLeafValue(module string, key []byte, value []byte) string { // Check for IO event tags if len(module) > 8 && module[:8] == "io_event" { - // Event tag value is typically CBOR var decoded interface{} if err := cbor.Unmarshal(value, &decoded); err == nil { - return fmt.Sprintf("event_value=%s", formatCBOR(decoded, len(value))) + info.ValueType = "io_event" + info.DecodedValue = formatCBOR(decoded, len(value)) + return info } - return fmt.Sprintf("event_value=binary(%d bytes)", len(value)) + info.ValueType = "binary" + info.BinarySize = len(value) + return info } // Try CBOR decode for regular state keys var decoded interface{} if err := cbor.Unmarshal(value, &decoded); err == nil { - return formatCBOR(decoded, len(value)) + info.ValueType = "cbor" + info.DecodedValue = formatCBOR(decoded, len(value)) + return info + } + + info.ValueType = "binary" + info.BinarySize = len(value) + return info +} + +// decodeEVMData decodes EVM module storage data. +// See: _oasis-sdk/runtime-sdk/modules/evm/src/state.rs +func decodeEVMData(module string, key []byte, value []byte) *EVMDataInfo { + // Module format: "evm:subtype" where subtype is extracted from key prefix + // The full key after module name starts with the storage type prefix + + // Find where module name ends in the key + moduleNameEnd := 0 + for i, b := range key { + if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' { + moduleNameEnd = i + 1 + } else { + break + } + } + + if moduleNameEnd >= len(key) { + return nil // No data after module name + } + + subKey := key[moduleNameEnd:] + if len(subKey) < 1 { + return nil + } + + evmInfo := &EVMDataInfo{} + storagePrefix := subKey[0] + data := subKey[1:] + + switch storagePrefix { + case 0x01: // CODES: evm + 0x01 + H160 (address) + evmInfo.StorageType = "code" + if len(data) >= 20 { + evmInfo.Address = fmt.Sprintf("%x", data[:20]) + data = data[20:] + } + // Value is contract bytecode + evmInfo.CodeSize = len(value) + if len(value) > 0 { + previewLen := 32 + if len(value) < previewLen { + previewLen = len(value) + } + evmInfo.CodePreview = fmt.Sprintf("%x", value[:previewLen]) + } + + case 0x02: // STORAGES: evm + 0x02 + H160 (address) + H256 (slot) + evmInfo.StorageType = "storage" + if len(data) >= 20 { + evmInfo.Address = fmt.Sprintf("%x", data[:20]) + data = data[20:] + if len(data) >= 32 { + evmInfo.StorageSlot = fmt.Sprintf("%x", data[:32]) + } + } + // Value is H256 storage value + if len(value) == 32 { + evmInfo.StorageValue = fmt.Sprintf("%x", value) + } + + case 0x03: // BLOCK_HASHES: evm + 0x03 + Round (uint64 BE) + evmInfo.StorageType = "block_hash" + if len(data) >= 8 { + evmInfo.Round = binary.BigEndian.Uint64(data[:8]) + } + // Value is H256 block hash + if len(value) == 32 { + evmInfo.BlockHash = fmt.Sprintf("%x", value) + } + + case 0x04: // CONFIDENTIAL_STORAGES: evm + 0x04 + H160 (address) + H256 (slot) + evmInfo.StorageType = "confidential_storage" + if len(data) >= 20 { + evmInfo.Address = fmt.Sprintf("%x", data[:20]) + data = data[20:] + if len(data) >= 32 { + evmInfo.StorageSlot = fmt.Sprintf("%x", data[:32]) + } + } + // Value is encrypted - we can only show size + if len(value) > 0 { + evmInfo.StorageValue = fmt.Sprintf("", len(value)) + } + + default: + return nil // Unknown EVM storage type } - return fmt.Sprintf("binary(%d bytes)", len(value)) + return evmInfo } diff --git a/badgerdb-sampler/main.go b/badgerdb-sampler/main.go index bf05c27..6f2ce85 100644 --- a/badgerdb-sampler/main.go +++ b/badgerdb-sampler/main.go @@ -16,14 +16,14 @@ var BadgerVersion string // Sample contains both raw and decoded key/value information type Sample struct { - RawKey string `json:"raw_key"` - RawKeySize int `json:"raw_key_size"` - KeyType string `json:"key_type"` - DecodedKey string `json:"decoded_key"` - RawValue string `json:"raw_value"` - RawValueSize int `json:"raw_value_size"` - DecodedValue string `json:"decoded_value"` - Timestamp int64 `json:"timestamp,omitempty"` + RawKey string `json:"raw_key"` + RawKeySize int `json:"raw_key_size"` + KeyType string `json:"key_type"` + DecodedKey interface{} `json:"decoded_key"` + RawValue string `json:"raw_value"` + RawValueSize int `json:"raw_value_size"` + DecodedValue interface{} `json:"decoded_value"` + Timestamp int64 `json:"timestamp,omitempty"` } // DBStats contains database statistics and samples @@ -175,49 +175,49 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { RawKey: fmt.Sprintf("%x", key), RawKeySize: len(key), KeyType: "unknown", - DecodedKey: "", + DecodedKey: nil, } // Decode key based on database type switch stats.DatabaseType { case "consensus-blockstore": - keyType, decodedKey := decodeKeyConsensusBlockstore(key) - sample.KeyType = keyType - sample.DecodedKey = decodedKey + keyInfo := decodeKeyConsensusBlockstore(key) + sample.KeyType = keyInfo.KeyType + sample.DecodedKey = keyInfo case "consensus-evidence": - keyType, decodedKey := decodeKeyConsensusEvidence(key) - sample.KeyType = keyType - sample.DecodedKey = decodedKey + keyInfo := decodeKeyConsensusEvidence(key) + sample.KeyType = keyInfo.KeyType + sample.DecodedKey = keyInfo case "consensus-mkvs": - keyType, decodedKey := decodeKeyConsensusMkvs(key) - sample.KeyType = keyType - sample.DecodedKey = decodedKey + keyInfo := decodeKeyConsensusMkvs(key) + sample.KeyType = keyInfo.KeyType + sample.DecodedKey = keyInfo case "consensus-state": - keyType, decodedKey := decodeKeyConsensusState(key) - sample.KeyType = keyType - sample.DecodedKey = decodedKey + keyInfo := decodeKeyConsensusState(key) + sample.KeyType = keyInfo.KeyType + sample.DecodedKey = keyInfo case "runtime-mkvs": - keyType, decodedKey := decodeKeyRuntimeMkvs(key) - sample.KeyType = keyType - sample.DecodedKey = decodedKey + keyInfo := decodeKeyRuntimeMkvs(key) + sample.KeyType = keyInfo.KeyType + sample.DecodedKey = keyInfo case "runtime-history": - keyType, decodedKey := decodeKeyRuntimeHistory(key) - sample.KeyType = keyType - sample.DecodedKey = decodedKey + keyInfo := decodeKeyRuntimeHistory(key) + sample.KeyType = keyInfo.KeyType + sample.DecodedKey = keyInfo } // Fetch and decode value err := item.Value(func(val []byte) error { sample.RawValueSize = len(val) - sample.RawValue = truncateHex(val, 100) - sample.DecodedValue = "" + sample.RawValue = truncateHex(val, 200) + sample.DecodedValue = nil // Decode value based on database type switch stats.DatabaseType { case "consensus-blockstore": - decoded, ts := decodeValueConsensusBlockstore(sample.KeyType, val) - sample.DecodedValue = decoded - sample.Timestamp = ts + valueInfo := decodeValueConsensusBlockstore(sample.KeyType, val) + sample.DecodedValue = valueInfo + sample.Timestamp = valueInfo.Timestamp case "consensus-evidence": sample.DecodedValue = decodeValueConsensusEvidence(sample.KeyType, val) case "consensus-mkvs": diff --git a/badgerdb-sampler/types_consensus.go b/badgerdb-sampler/types_consensus.go new file mode 100644 index 0000000..5b9047f --- /dev/null +++ b/badgerdb-sampler/types_consensus.go @@ -0,0 +1,152 @@ +package main + +// Consensus decode output types - structured representations of decoded consensus database entries. +// These types separate decoding logic from string formatting, enabling flexible output formats. + +// ============================================================================= +// Blockstore Types +// ============================================================================= + +// ConsensusBlockstoreKeyInfo represents a decoded consensus-blockstore key. +// See: tendermint/store/store.go for key formats (H:, P:, C:, SC:, BH:) +type ConsensusBlockstoreKeyInfo struct { + KeyType string `json:"key_type"` // "blockstore_state", "block_meta", "block_part", "block_commit", "seen_commit", "block_hash", "unknown" + Height int64 `json:"height,omitempty"` // For H:, C:, SC:, P: keys + PartIndex int `json:"part_index,omitempty"` // For P: keys + Hash string `json:"hash,omitempty"` // For BH: keys + RawKey string `json:"raw_key,omitempty"` // Original ASCII key +} + +// ConsensusBlockstoreValueInfo represents a decoded consensus-blockstore value. +// See: tendermint/proto/tendermint/store/types.proto (BlockStoreState) +// See: tendermint/proto/tendermint/types/types.proto (BlockMeta, Part, Commit) +type ConsensusBlockstoreValueInfo struct { + KeyType string `json:"key_type"` + Size int `json:"size"` + Timestamp int64 `json:"timestamp,omitempty"` // Unix timestamp from block meta + State *ConsensusBlockStoreState `json:"state,omitempty"` + BlockMeta *ConsensusBlockMetaInfo `json:"block_meta,omitempty"` + Part *ConsensusPartInfo `json:"part,omitempty"` + Commit *ConsensusCommitInfo `json:"commit,omitempty"` + HashHeight int64 `json:"hash_height,omitempty"` // For block_hash type + DecodeError string `json:"decode_error,omitempty"` +} + +// ConsensusBlockStoreState represents BlockStoreState from tendermint. +// See: tendermint/proto/tendermint/store/types.proto (BlockStoreState message) +type ConsensusBlockStoreState struct { + Base int64 `json:"base"` + Height int64 `json:"height"` +} + +// ConsensusBlockMetaInfo represents decoded block metadata. +// See: tendermint/proto/tendermint/types/types.proto (BlockMeta, Header messages) +type ConsensusBlockMetaInfo struct { + Height int64 `json:"height"` + Time string `json:"time"` // RFC3339 format + ChainID string `json:"chain_id"` + NumTxs int64 `json:"num_txs"` + AppHash string `json:"app_hash,omitempty"` // hex, truncated +} + +// ConsensusPartInfo represents decoded block part. +// See: tendermint/proto/tendermint/types/types.proto (Part message) +type ConsensusPartInfo struct { + Index uint32 `json:"index"` + BytesSize int `json:"bytes_size"` + ProofTotal int64 `json:"proof_total"` +} + +// ConsensusCommitInfo represents decoded commit. +// See: tendermint/proto/tendermint/types/types.proto (Commit message) +type ConsensusCommitInfo struct { + Height int64 `json:"height"` + Round int32 `json:"round"` + Signatures int `json:"signatures"` +} + +// ============================================================================= +// Evidence Types +// ============================================================================= + +// ConsensusEvidenceKeyInfo represents a decoded consensus-evidence key. +// See: tendermint/store/evidence/pool.go for key formats +type ConsensusEvidenceKeyInfo struct { + KeyType string `json:"key_type"` + PrefixByte byte `json:"prefix_byte,omitempty"` + RawKey string `json:"raw_key,omitempty"` // hex +} + +// ConsensusEvidenceValueInfo represents a decoded consensus-evidence value. +// See: tendermint/proto/tendermint/types/evidence.proto (DuplicateVoteEvidence) +type ConsensusEvidenceValueInfo struct { + Size int `json:"size"` + VoteAHeight int64 `json:"vote_a_height,omitempty"` + VoteBHeight int64 `json:"vote_b_height,omitempty"` + DecodeError string `json:"decode_error,omitempty"` +} + +// ============================================================================= +// MKVS Types +// ============================================================================= + +// ConsensusMkvsKeyInfo represents a decoded consensus-mkvs key. +// See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 +type ConsensusMkvsKeyInfo struct { + KeyType string `json:"key_type"` // "node", "write_log", "roots_metadata", "root_updated_nodes", "metadata", "multipart_restore_log", "root_node", "unknown" + Height uint64 `json:"height,omitempty"` // For write_log, roots_metadata, root_updated_nodes + Hash string `json:"hash,omitempty"` // hex, truncated + RootType byte `json:"root_type,omitempty"` + DbPrefix byte `json:"db_prefix,omitempty"` // 0x01 or 0x05 if present + DecodeError string `json:"decode_error,omitempty"` +} + +// ConsensusMkvsNodeInfo represents a decoded consensus-mkvs value (node). +// See: _oasis-core/go/storage/mkvs/node/node.go:26-32 (prefixes), 294-309 (InternalNode), 531-537 (LeafNode) +type ConsensusMkvsNodeInfo struct { + NodeType string `json:"node_type"` // "leaf", "internal", "nil", "non_node", "unknown" + Size int `json:"size"` + Leaf *ConsensusMkvsLeafInfo `json:"leaf,omitempty"` + Internal *ConsensusMkvsInternalInfo `json:"internal,omitempty"` + DecodeError string `json:"decode_error,omitempty"` +} + +// ConsensusMkvsLeafInfo represents a decoded MKVS LeafNode for consensus. +// See: _oasis-core/go/storage/mkvs/node/node.go:531-537 (LeafNode) +// See: _oasis-core/go/consensus/tendermint/apps/*/state/state.go (module prefixes) +type ConsensusMkvsLeafInfo struct { + Module string `json:"module"` + KeyLen int `json:"key_len"` + Key string `json:"key,omitempty"` // hex, truncated + ValueLen int `json:"value_len"` + DecodedValue interface{} `json:"decoded_value,omitempty"` // CBOR decoded or size info +} + +// ConsensusMkvsInternalInfo represents a decoded MKVS InternalNode. +// See: _oasis-core/go/storage/mkvs/node/node.go:294-309 (InternalNode) +type ConsensusMkvsInternalInfo struct { + LabelBits uint16 `json:"label_bits"` + HasLeaf bool `json:"has_leaf"` + LeftHash string `json:"left_hash,omitempty"` // hex, truncated + RightHash string `json:"right_hash,omitempty"` // hex, truncated +} + +// ============================================================================= +// State Types +// ============================================================================= + +// ConsensusStateKeyInfo represents a decoded consensus-state key. +// See: tendermint/state/store.go for key formats (abciResponsesKey, validatorsKey, etc.) +type ConsensusStateKeyInfo struct { + KeyType string `json:"key_type"` // "abci_responses", "consensus_params", "validators", "state", "genesis", "text_key", "binary", "unknown" + DecodedKey string `json:"decoded_key,omitempty"` + IsBinary bool `json:"is_binary,omitempty"` +} + +// ConsensusStateValueInfo represents a decoded consensus-state value. +// See: tendermint/proto/tendermint/state/types.proto (various state types) +// Note: Tendermint state values are complex protobuf - we just report size. +type ConsensusStateValueInfo struct { + KeyType string `json:"key_type"` + Size int `json:"size"` +} diff --git a/badgerdb-sampler/types_runtime.go b/badgerdb-sampler/types_runtime.go new file mode 100644 index 0000000..8864269 --- /dev/null +++ b/badgerdb-sampler/types_runtime.go @@ -0,0 +1,195 @@ +package main + +import "github.com/fxamacker/cbor/v2" + +// Runtime decode output types - structured representations of decoded runtime database entries. +// These types separate decoding logic from string formatting, enabling flexible output formats. + +// ============================================================================= +// CBOR Deserialization Types (for unmarshaling from database) +// ============================================================================= + +// RuntimeHistoryMetadata represents the metadata stored in runtime history DB. +// See: _oasis-core/go/runtime/history/db.go:34-44 (dbMetadata) +type RuntimeHistoryMetadata struct { + RuntimeID []byte `cbor:"runtime_id"` + Version uint64 `cbor:"version"` + LastConsensusHeight int64 `cbor:"last_consensus_height"` + LastRound uint64 `cbor:"last_round"` +} + +// RuntimeHistoryAnnotatedBlock represents an annotated block in runtime history. +// See: _oasis-core/go/roothash/api/api.go:401-409 (AnnotatedBlock) +type RuntimeHistoryAnnotatedBlock struct { + Height int64 `cbor:"consensus_height"` + Block *RuntimeHistoryBlock `cbor:"block"` +} + +// RuntimeHistoryBlock represents a runtime block. +// See: _oasis-core/go/roothash/api/block/block.go:7-12 (Block) +type RuntimeHistoryBlock struct { + Header RuntimeHistoryBlockHeader `cbor:"header"` +} + +// RuntimeHistoryBlockHeader represents a runtime block header. +// See: _oasis-core/go/roothash/api/block/header.go:69-99 (Header) +type RuntimeHistoryBlockHeader struct { + Version uint16 `cbor:"version"` + Namespace []byte `cbor:"namespace"` + Round uint64 `cbor:"round"` + Timestamp uint64 `cbor:"timestamp"` // POSIX time (Unix seconds) + HeaderType uint8 `cbor:"header_type"` + PreviousHash []byte `cbor:"previous_hash"` + IORoot []byte `cbor:"io_root"` + StateRoot []byte `cbor:"state_root"` + MessagesHash []byte `cbor:"messages_hash"` + InMessagesHash []byte `cbor:"in_msgs_hash"` +} + +// RuntimeHistoryRoundResults represents round results in runtime history. +// See: _oasis-core/go/roothash/api/results.go:5-17 (RoundResults) +type RuntimeHistoryRoundResults struct { + Messages []RuntimeHistoryMessageEvent `cbor:"messages,omitempty"` + GoodComputeEntities [][]byte `cbor:"good_compute_entities,omitempty"` + BadComputeEntities [][]byte `cbor:"bad_compute_entities,omitempty"` +} + +// RuntimeHistoryMessageEvent represents a message event. +// See: _oasis-core/go/roothash/api/api.go:492-499 (MessageEvent) +type RuntimeHistoryMessageEvent struct { + Module string `cbor:"module,omitempty"` + Code uint32 `cbor:"code,omitempty"` + Index uint32 `cbor:"index,omitempty"` + Result cbor.RawMessage `cbor:"result,omitempty"` +} + +// RuntimeInputArtifacts represents input transaction artifacts stored in IO tree. +// See: _oasis-core/go/runtime/transaction/transaction.go:129-140 (inputArtifacts) +type RuntimeInputArtifacts struct { + _ struct{} `cbor:",toarray"` + Input []byte + BatchOrder uint32 +} + +// RuntimeOutputArtifacts represents output transaction artifacts stored in IO tree. +// See: _oasis-core/go/runtime/transaction/transaction.go:145-150 (outputArtifacts) +type RuntimeOutputArtifacts struct { + _ struct{} `cbor:",toarray"` + Output []byte +} + +// ============================================================================= +// Decoded Output Types (structured representations for JSON output) +// ============================================================================= + +// RuntimeMkvsKeyInfo represents a decoded runtime-mkvs key. +// See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 +type RuntimeMkvsKeyInfo struct { + KeyType string `json:"key_type"` // "node", "write_log", "roots_metadata", "root_updated_nodes", "metadata", "unknown" + Height uint64 `json:"height,omitempty"` // For write_log, roots_metadata, root_updated_nodes + Hash string `json:"hash,omitempty"` // For node (hex, truncated) + DbPrefix byte `json:"db_prefix,omitempty"` // 0x01 or 0x05 if present + DecodeError string `json:"decode_error,omitempty"` +} + +// RuntimeMkvsNodeInfo represents a decoded runtime-mkvs value (node). +// See: _oasis-core/go/storage/mkvs/node/node.go:26-32 (prefixes), 294-309 (InternalNode), 531-537 (LeafNode) +type RuntimeMkvsNodeInfo struct { + NodeType string `json:"node_type"` // "leaf", "internal", "nil", "non_node", "unknown" + Size int `json:"size"` + Leaf *RuntimeMkvsLeafInfo `json:"leaf,omitempty"` + Internal *RuntimeMkvsInternalInfo `json:"internal,omitempty"` + DecodeError string `json:"decode_error,omitempty"` +} + +// RuntimeMkvsLeafInfo represents a decoded MKVS LeafNode. +// See: _oasis-core/go/storage/mkvs/node/node.go:531-537 +type RuntimeMkvsLeafInfo struct { + Module string `json:"module"` + KeyLen int `json:"key_len"` + Key string `json:"key,omitempty"` // hex, truncated + ValueLen int `json:"value_len"` + DecodedValue *RuntimeLeafValueInfo `json:"decoded_value,omitempty"` +} + +// RuntimeMkvsInternalInfo represents a decoded MKVS InternalNode. +// See: _oasis-core/go/storage/mkvs/node/node.go:294-309 +type RuntimeMkvsInternalInfo struct { + LabelBits uint16 `json:"label_bits"` + HasLeaf bool `json:"has_leaf"` + LeftHash string `json:"left_hash,omitempty"` // hex, truncated + RightHash string `json:"right_hash,omitempty"` // hex, truncated +} + +// RuntimeHistoryKeyInfo represents a decoded runtime-history key. +// See: _oasis-core/go/runtime/history/db.go:19-31 +type RuntimeHistoryKeyInfo struct { + KeyType string `json:"key_type"` // "metadata", "block", "round_results", "unknown" + Height uint64 `json:"height,omitempty"` // For block, round_results (block height/round) + ExtraData string `json:"extra,omitempty"` // Unexpected extra bytes (hex) + DecodeError string `json:"decode_error,omitempty"` +} + +// RuntimeHistoryValueInfo represents a decoded runtime-history value. +type RuntimeHistoryValueInfo struct { + KeyType string `json:"key_type"` + Size int `json:"size"` + Metadata *RuntimeHistoryMetadataInfo `json:"metadata,omitempty"` + Block *RuntimeHistoryBlockInfo `json:"block,omitempty"` + RoundResults *RuntimeHistoryRoundResultsInfo `json:"round_results,omitempty"` + DecodeError string `json:"decode_error,omitempty"` +} + +// RuntimeHistoryMetadataInfo represents decoded runtime history metadata. +// See: _oasis-core/go/runtime/history/db.go:34-44 +type RuntimeHistoryMetadataInfo struct { + Version uint64 `json:"version"` + RuntimeID string `json:"runtime_id"` // hex, truncated + LastRound uint64 `json:"last_round"` + LastConsensusHeight int64 `json:"last_consensus_height"` +} + +// RuntimeHistoryBlockInfo represents a decoded runtime history block. +// See: _oasis-core/go/roothash/api/api.go:402-409 (AnnotatedBlock) +// See: _oasis-core/go/roothash/api/block/header.go:69-99 (Header) +type RuntimeHistoryBlockInfo struct { + ConsensusHeight int64 `json:"consensus_height"` + Round uint64 `json:"round"` + Timestamp string `json:"timestamp"` // RFC3339 format + HeaderType string `json:"header_type"` + StateRoot string `json:"state_root,omitempty"` // hex, truncated + BlockNil bool `json:"block_nil,omitempty"` // true if block was nil +} + +// RuntimeHistoryRoundResultsInfo represents decoded runtime history round results. +// See: _oasis-core/go/roothash/api/results.go:5-17 +type RuntimeHistoryRoundResultsInfo struct { + MessageCount int `json:"message_count"` + GoodComputeEntities int `json:"good_compute_entities"` + BadComputeEntities int `json:"bad_compute_entities"` +} + +// RuntimeLeafValueInfo represents a decoded MKVS leaf node value. +// See: _oasis-core/go/runtime/transaction/transaction.go:129-150 (artifacts) +type RuntimeLeafValueInfo struct { + ValueType string `json:"value_type"` // "io_input", "io_output", "io_event", "evm_code", "evm_storage", "evm_block_hash", "cbor", "binary" + InputSize int `json:"input_size,omitempty"` + BatchOrder uint32 `json:"batch_order,omitempty"` + OutputSize int `json:"output_size,omitempty"` + DecodedValue interface{} `json:"decoded_value,omitempty"` // For CBOR/event values + BinarySize int `json:"binary_size,omitempty"` // For binary values + EVM *EVMDataInfo `json:"evm,omitempty"` // For EVM-specific data +} + +// EVMDataInfo represents decoded EVM storage data. +// See: _oasis-sdk/runtime-sdk/modules/evm/src/state.rs +type EVMDataInfo struct { + StorageType string `json:"storage_type"` // "code", "storage", "block_hash", "confidential_storage" + Address string `json:"address,omitempty"` // H160 (20 bytes) - contract address (hex) + StorageSlot string `json:"storage_slot,omitempty"` // H256 (32 bytes) - storage slot (hex) + StorageValue string `json:"storage_value,omitempty"` // H256 (32 bytes) - storage value (hex) + Round uint64 `json:"round,omitempty"` // For block_hash type + BlockHash string `json:"block_hash,omitempty"` // H256 (32 bytes) - block hash (hex) + CodeSize int `json:"code_size,omitempty"` // Size of contract bytecode + CodePreview string `json:"code_preview,omitempty"` // First few bytes of bytecode (hex) +} diff --git a/badgerdb-sampler/utils.go b/badgerdb-sampler/utils.go index 8cd0b10..6316f58 100644 --- a/badgerdb-sampler/utils.go +++ b/badgerdb-sampler/utils.go @@ -192,14 +192,6 @@ func isPrintableASCII(s string) bool { return true } -// truncateBytes returns first n bytes or all bytes if shorter -func truncateBytes(data []byte, n int) []byte { - if len(data) <= n { - return data - } - return data[:n] -} - // truncateHex converts bytes to hex and truncates if too long func truncateHex(data []byte, maxLen int) string { hex := fmt.Sprintf("%x", data) @@ -208,3 +200,4 @@ func truncateHex(data []byte, maxLen int) string { } return hex } + From 024a3d5f9f90d6f5ff0bc7523e9637366aad6e4d Mon Sep 17 00:00:00 2001 From: gw0 Date: Mon, 24 Nov 2025 13:31:57 +0100 Subject: [PATCH 12/22] badgerdb-sampler: Add decoding of EVM events, transactions and calls --- badgerdb-sampler/decode_evm.go | 274 +++++++++++++++++++++++++++++ badgerdb-sampler/decode_runtime.go | 123 ++----------- badgerdb-sampler/evm_signatures.go | 63 +++++++ badgerdb-sampler/types_evm.go | 60 +++++++ badgerdb-sampler/types_runtime.go | 30 ++-- badgerdb-sampler/utils.go | 10 ++ 6 files changed, 435 insertions(+), 125 deletions(-) create mode 100644 badgerdb-sampler/decode_evm.go create mode 100644 badgerdb-sampler/evm_signatures.go create mode 100644 badgerdb-sampler/types_evm.go diff --git a/badgerdb-sampler/decode_evm.go b/badgerdb-sampler/decode_evm.go new file mode 100644 index 0000000..235582f --- /dev/null +++ b/badgerdb-sampler/decode_evm.go @@ -0,0 +1,274 @@ + +package main + +import ( + "encoding/binary" + "fmt" + + "github.com/fxamacker/cbor/v2" +) + +// decodeEVMData decodes EVM module storage data. +// See: _oasis-sdk/runtime-sdk/modules/evm/src/state.rs +func decodeEVMData(module string, key []byte, value []byte) *EVMDataInfo { + // Module format: "evm:subtype" where subtype is extracted from key prefix + // The full key after module name starts with the storage type prefix + + // Find where module name ends in the key + moduleNameEnd := 0 + for i, b := range key { + if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' { + moduleNameEnd = i + 1 + } else { + break + } + } + + if moduleNameEnd >= len(key) { + return nil // No data after module name + } + + subKey := key[moduleNameEnd:] + if len(subKey) < 1 { + return nil + } + + evmInfo := &EVMDataInfo{} + storagePrefix := subKey[0] + data := subKey[1:] + + switch storagePrefix { + case 0x01: // CODES: evm + 0x01 + H160 (address) + evmInfo.StorageType = "code" + if len(data) >= 20 { + evmInfo.Address = fmt.Sprintf("%x", data[:20]) + data = data[20:] + } + // Value is contract bytecode + evmInfo.BytecodeSize = len(value) + if len(value) > 0 { + previewLen := 32 + if len(value) < previewLen { + previewLen = len(value) + } + evmInfo.BytecodeRaw = fmt.Sprintf("%x", value[:previewLen]) + } + + case 0x02: // STORAGES: evm + 0x02 + H160 (address) + H256 (slot) + evmInfo.StorageType = "storage" + if len(data) >= 20 { + evmInfo.Address = fmt.Sprintf("%x", data[:20]) + data = data[20:] + if len(data) >= 32 { + evmInfo.StorageSlot = fmt.Sprintf("%x", data[:32]) + } + } + // Value is H256 storage value + if len(value) == 32 { + evmInfo.StorageValue = fmt.Sprintf("%x", value) + } + + case 0x03: // BLOCK_HASHES: evm + 0x03 + Round (uint64 BE) + evmInfo.StorageType = "block_hash" + if len(data) >= 8 { + evmInfo.Round = binary.BigEndian.Uint64(data[:8]) + } + // Value is H256 block hash + if len(value) == 32 { + evmInfo.BlockHash = fmt.Sprintf("%x", value) + } + + case 0x04: // CONFIDENTIAL_STORAGES: evm + 0x04 + H160 (address) + H256 (slot) + evmInfo.StorageType = "confidential_storage" + if len(data) >= 20 { + evmInfo.Address = fmt.Sprintf("%x", data[:20]) + data = data[20:] + if len(data) >= 32 { + evmInfo.StorageSlot = fmt.Sprintf("%x", data[:32]) + } + } + // Value is encrypted - we can only show size + if len(value) > 0 { + evmInfo.StorageValue = fmt.Sprintf("", len(value)) + } + + default: + return nil // Unknown EVM storage type + } + + return evmInfo +} + +// decodeEVMEvent decodes an EVM Log event from CBOR-encoded value. +// See: _oasis-sdk/runtime-sdk/modules/evm/src/lib.rs:253-263 +func decodeEVMEvent(value []byte) *EVMEventInfo { + info := &EVMEventInfo{} + + // Value is CBOR: [event_code, {address, topics, data}] + var eventWrapper []interface{} + if err := cbor.Unmarshal(value, &eventWrapper); err != nil { + info.DecodeError = err.Error() + return info + } + if len(eventWrapper) < 2 { + info.DecodeError = "invalid format" + return info + } + + eventData, ok := eventWrapper[1].(map[interface{}]interface{}) + if !ok { + info.DecodeError = "invalid format" + return info + } + + // Extract address (H160) + if addrBytes, ok := eventData["address"].([]byte); ok && len(addrBytes) == 20 { + info.Address = fmt.Sprintf("%x", addrBytes) + } else { + // Continue decoding even with invalid address - topics and data may still be useful + info.DecodeError = "invalid address (continuing decode)" + } + + // Extract topics (Vec) + if topicsArray, ok := eventData["topics"].([]interface{}); ok { + info.TopicCount = len(topicsArray) + for i, topic := range topicsArray { + if topicBytes, ok := topic.([]byte); ok && len(topicBytes) == 32 { + topicHex := fmt.Sprintf("%x", topicBytes) + info.Topics = append(info.Topics, topicHex) + if i == 0 { + info.EventHash = topicHex + if sig, found := EVMEventSignatures[topicHex]; found { + info.EventSignature = sig + } + } + } + } + } + + // Extract data + if dataBytes, ok := eventData["data"].([]byte); ok { + info.DataSize = len(dataBytes) + info.DataRaw = truncateHex(dataBytes, 128) // 64 bytes = 128 hex chars + } + + return info +} + +// decodeEVMTxInput decodes EVM transaction input artifacts. +func decodeEVMTxInput(key []byte, value []byte) *EVMTxInputInfo { + info := &EVMTxInputInfo{InputSize: len(value)} + + if len(key) >= 33 { + info.TxHash = fmt.Sprintf("%x", key[1:33]) + } + + var ia RuntimeInputArtifacts + if err := cbor.Unmarshal(value, &ia); err != nil { + info.DecodeError = err.Error() + return info + } + info.BatchOrder = ia.BatchOrder + + var call map[interface{}]interface{} + if err := cbor.Unmarshal(ia.Input, &call); err == nil { + if methodVal, ok := call["method"].(string); ok { + info.Method = methodVal + if methodVal == "evm.Call" || methodVal == "evm.Create" { + info.EVMCall = decodeEVMCallBody(call, methodVal == "evm.Create") + } + } + } + + return info +} + +// decodeEVMCallBody decodes EVM call/create body from SDK Call structure. +func decodeEVMCallBody(call map[interface{}]interface{}, isCreate bool) *EVMCallInfo { + info := &EVMCallInfo{} + if isCreate { + info.Type = "create" + } else { + info.Type = "call" + } + + bodyBytes, ok := call["body"].([]byte) + if !ok { + return info + } + + var bodyMap map[interface{}]interface{} + if err := cbor.Unmarshal(bodyBytes, &bodyMap); err != nil { + return info + } + + // Extract address (calls only) + if !isCreate { + if addrBytes, ok := bodyMap["address"].([]byte); ok && len(addrBytes) == 20 { + info.Address = fmt.Sprintf("%x", addrBytes) + } + } + + // Extract value + if valueBytes, ok := bodyMap["value"].([]byte); ok { + info.Value = formatU256(valueBytes) + } + + // Extract data/init_code + dataKey := "data" + if isCreate { + dataKey = "init_code" + } + if dataBytes, ok := bodyMap[dataKey].([]byte); ok { + info.DataSize = len(dataBytes) + info.DataRaw = truncateHex(dataBytes, 64) // 32 bytes = 64 hex chars + } + + return info +} + +// decodeEVMTxOutput decodes EVM transaction output artifacts. +func decodeEVMTxOutput(value []byte) *EVMTxOutputInfo { + info := &EVMTxOutputInfo{OutputSize: len(value)} + + var oa RuntimeOutputArtifacts + if err := cbor.Unmarshal(value, &oa); err != nil { + info.DecodeError = err.Error() + return info + } + + if len(oa.Output) == 0 { + info.Success = true + return info + } + + // Try to decode as CBOR CallResult + var result map[interface{}]interface{} + if err := cbor.Unmarshal(oa.Output, &result); err != nil { + // Raw bytes - success + info.Success = true + info.ResultSize = len(oa.Output) + info.ResultRaw = truncateHex(oa.Output, 64) // 32 bytes = 64 hex chars + return info + } + + // Check success/failure + if okVal, exists := result["ok"]; exists { + info.Success = true + if okBytes, ok := okVal.([]byte); ok { + info.ResultSize = len(okBytes) + info.ResultRaw = truncateHex(okBytes, 64) + } + } else if failVal, exists := result["fail"]; exists { + info.Success = false + if failMap, ok := failVal.(map[interface{}]interface{}); ok { + if msgVal, ok := failMap["message"].(string); ok { + info.Error = msgVal + } else { + info.Error = fmt.Sprintf("%v", failMap) + } + } + } + + return info +} diff --git a/badgerdb-sampler/decode_runtime.go b/badgerdb-sampler/decode_runtime.go index f19880b..cdc5c56 100644 --- a/badgerdb-sampler/decode_runtime.go +++ b/badgerdb-sampler/decode_runtime.go @@ -260,7 +260,7 @@ func decodeValueRuntimeHistory(keyType string, value []byte) RuntimeHistoryValue } info.Metadata = &RuntimeHistoryMetadataInfo{ Version: meta.Version, - RuntimeID: truncateHex(meta.RuntimeID, 16), + RuntimeID: fmt.Sprintf("%x", meta.RuntimeID), LastRound: meta.LastRound, LastConsensusHeight: meta.LastConsensusHeight, } @@ -354,27 +354,31 @@ func decodeLeafValue(module string, key []byte, value []byte) *RuntimeLeafValueI kind := key[33] if kind == 1 { // Input artifact - var ia RuntimeInputArtifacts - if err := cbor.Unmarshal(value, &ia); err == nil { - info.ValueType = "io_input" - info.InputSize = len(ia.Input) - info.BatchOrder = ia.BatchOrder - return info - } + info.ValueType = "io_input" + info.EVMTxInput = decodeEVMTxInput(key, value) + return info } else if kind == 2 { // Output artifact - var oa RuntimeOutputArtifacts - if err := cbor.Unmarshal(value, &oa); err == nil { - info.ValueType = "io_output" - info.OutputSize = len(oa.Output) - return info - } + info.ValueType = "io_output" + info.EVMTxOutput = decodeEVMTxOutput(value) + return info } } } // Check for IO event tags if len(module) > 8 && module[:8] == "io_event" { + // Special handling for EVM events + if len(module) >= 12 && module[9:12] == "evm" { + evmEvent := decodeEVMEvent(value) + if evmEvent != nil { + info.ValueType = "evm_event" + info.EVMEvent = evmEvent + return info + } + } + + // Generic event decoding for non-EVM events var decoded interface{} if err := cbor.Unmarshal(value, &decoded); err == nil { info.ValueType = "io_event" @@ -398,94 +402,3 @@ func decodeLeafValue(module string, key []byte, value []byte) *RuntimeLeafValueI info.BinarySize = len(value) return info } - -// decodeEVMData decodes EVM module storage data. -// See: _oasis-sdk/runtime-sdk/modules/evm/src/state.rs -func decodeEVMData(module string, key []byte, value []byte) *EVMDataInfo { - // Module format: "evm:subtype" where subtype is extracted from key prefix - // The full key after module name starts with the storage type prefix - - // Find where module name ends in the key - moduleNameEnd := 0 - for i, b := range key { - if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' { - moduleNameEnd = i + 1 - } else { - break - } - } - - if moduleNameEnd >= len(key) { - return nil // No data after module name - } - - subKey := key[moduleNameEnd:] - if len(subKey) < 1 { - return nil - } - - evmInfo := &EVMDataInfo{} - storagePrefix := subKey[0] - data := subKey[1:] - - switch storagePrefix { - case 0x01: // CODES: evm + 0x01 + H160 (address) - evmInfo.StorageType = "code" - if len(data) >= 20 { - evmInfo.Address = fmt.Sprintf("%x", data[:20]) - data = data[20:] - } - // Value is contract bytecode - evmInfo.CodeSize = len(value) - if len(value) > 0 { - previewLen := 32 - if len(value) < previewLen { - previewLen = len(value) - } - evmInfo.CodePreview = fmt.Sprintf("%x", value[:previewLen]) - } - - case 0x02: // STORAGES: evm + 0x02 + H160 (address) + H256 (slot) - evmInfo.StorageType = "storage" - if len(data) >= 20 { - evmInfo.Address = fmt.Sprintf("%x", data[:20]) - data = data[20:] - if len(data) >= 32 { - evmInfo.StorageSlot = fmt.Sprintf("%x", data[:32]) - } - } - // Value is H256 storage value - if len(value) == 32 { - evmInfo.StorageValue = fmt.Sprintf("%x", value) - } - - case 0x03: // BLOCK_HASHES: evm + 0x03 + Round (uint64 BE) - evmInfo.StorageType = "block_hash" - if len(data) >= 8 { - evmInfo.Round = binary.BigEndian.Uint64(data[:8]) - } - // Value is H256 block hash - if len(value) == 32 { - evmInfo.BlockHash = fmt.Sprintf("%x", value) - } - - case 0x04: // CONFIDENTIAL_STORAGES: evm + 0x04 + H160 (address) + H256 (slot) - evmInfo.StorageType = "confidential_storage" - if len(data) >= 20 { - evmInfo.Address = fmt.Sprintf("%x", data[:20]) - data = data[20:] - if len(data) >= 32 { - evmInfo.StorageSlot = fmt.Sprintf("%x", data[:32]) - } - } - // Value is encrypted - we can only show size - if len(value) > 0 { - evmInfo.StorageValue = fmt.Sprintf("", len(value)) - } - - default: - return nil // Unknown EVM storage type - } - - return evmInfo -} diff --git a/badgerdb-sampler/evm_signatures.go b/badgerdb-sampler/evm_signatures.go new file mode 100644 index 0000000..a819753 --- /dev/null +++ b/badgerdb-sampler/evm_signatures.go @@ -0,0 +1,63 @@ +package main + +// EVMEventSignatures maps Keccak256 event signature hashes to human-readable signatures. +// The hash is already computed and stored in topics[0] by the EVM - we just look it up. +// Hashes are lowercase hex strings without "0x" prefix. +// +// This is a curated list focused on protocols deployed on Oasis Network (Sapphire and Emerald). +// For comprehensive coverage, consider integrating with external signature databases like +// 4byte.directory or Etherscan. Users can extend this map with project-specific signatures. +var EVMEventSignatures = map[string]string{ + // ERC-20 Token Standard (Used by all tokens on Oasis) + "ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef": "Transfer(address,address,uint256)", + "8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925": "Approval(address,address,uint256)", + + // ERC-721 NFT Standard + "17307eab39ab6107e8899845ad3d59bd9653f200f220920489ca2b5937696c31": "ApprovalForAll(address,address,bool)", + + // ERC-1155 Multi-Token Standard + "c3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62": "TransferSingle(address,address,address,uint256,uint256)", + "4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb": "TransferBatch(address,address,address,uint256[],uint256[])", + + // Ownable / Access Control (Common pattern in Oasis contracts) + "8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e0": "OwnershipTransferred(address,address)", + "2f8788117e7eff1d82e926ec794901d17c78024a50270940304540a733656f0d": "RoleGranted(bytes32,address,address)", + "f6391f5c32d9c69d2a47ea670b442974b53935d1edc7fd64eb21e047a839171b": "RoleRevoked(bytes32,address,address)", + + // Pausable (Common security pattern) + "62e78cea01bee320cd4e420270b5ea74000d11b0c9f74754ebdbfc544b05a258": "Paused(address)", + "5db9ee0a495bf2e6ff9c91a7834c1ba4fdd244a5e8aa4e537bd38aeae4b073aa": "Unpaused(address)", + + // Wrapped ETH/ROSE (wROSE: 0x8Bc2B030b299964eEfb5e1e0b36991352E56D2D3) + "e1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c": "Deposit(address,uint256)", + "7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65": "Withdrawal(address,uint256)", + + // Uniswap V2 / DEX Events (YuzuSwap, ValleySwap, LilacSwap on Emerald) + "1c411e9a96e071241c2f21f7726b17ae89e3cab4c78be50e062b03a9fffbbad1": "Sync(uint112,uint112)", + "d78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822": "Swap(address,uint256,uint256,uint256,uint256,address)", + "0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9": "PairCreated(address,address,address,uint256)", + "4c209b5fc8ad50758f13e2e1088ba56a560dff690a1c6fef26394f4c03821c4f": "Mint(address,uint256,uint256)", + "dccd412f0b1252819cb1fd330b93224ca42612892bb3f4f789976e6d81936496": "Burn(address,uint256,uint256,address)", + + // Proxy Patterns (EIP-1967) - Common upgrade pattern + "bc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b": "Upgraded(address)", + "7e644d79422f17c01e4894b5f4f588d331ebfa28653d42ae832dc59e38c9798f": "AdminChanged(address,address)", + "101b8081ff3b56bbf45deb824d86a3b0fd38b7e3dd42421105cf8abe9106db0b": "BeaconUpgraded(address)", + + // Bridge Events (Celer MessageBus: 0x9Bb46D5100d2Db4608112026951c9C965b233f4D, Wormhole) + "1591690b8638f5fb2dbec82ac741805ac5da8b45dc5263f4875b0496fdce4e05": "TokensBridged(address,address,uint256,uint256,uint256)", + + // Oasis Network Specific Notes: + // - Sapphire and Emerald ParaTimes are EVM-compatible and use standard EVM events + // - wROSE (Wrapped ROSE) uses standard WETH9 events listed above + // - YuzuSwap: First DEX on Emerald (Uniswap V2 fork) - 0x250d48C5E78f1E85F7AB07FEC61E93ba703aE668 + // - ValleySwap: DEX on Emerald - 0x7C0b0a525fc6A2caDf7AE37198119025C6feA28a + // - LilacSwap: DEX on Emerald - 0x5ca7e4301C9ac05B94e4c5b0C1812015f0A0be5f + // - Celer MessageBus: Cross-chain bridge - 0x9Bb46D5100d2Db4608112026951c9C965b233f4D + // - Band Oracle: Price feeds on Sapphire - 0xDA7a001b254CD22e46d3eAB04d937489c93174C3 + // - Sapphire DeFi: Midas, Neby, Thorn, Accumulated Finance, BitProtocol + // + // To add more signatures, examine contract ABIs on: + // - Sapphire: https://explorer.oasis.io/mainnet/sapphire + // - Emerald: https://explorer.emerald.oasis.dev +} diff --git a/badgerdb-sampler/types_evm.go b/badgerdb-sampler/types_evm.go new file mode 100644 index 0000000..a660bf6 --- /dev/null +++ b/badgerdb-sampler/types_evm.go @@ -0,0 +1,60 @@ +package main + +// EVMDataInfo represents decoded EVM storage data. +// See: _oasis-sdk/runtime-sdk/modules/evm/src/state.rs +type EVMDataInfo struct { + StorageType string `json:"storage_type"` // "code", "storage", "block_hash", "confidential_storage" + Address string `json:"address,omitempty"` // H160 (20 bytes) - contract address (hex) + StorageSlot string `json:"storage_slot,omitempty"` // H256 (32 bytes) - storage slot (hex) + StorageValue string `json:"storage_value,omitempty"` // H256 (32 bytes) - storage value (hex) + Round uint64 `json:"round,omitempty"` // For block_hash type + BlockHash string `json:"block_hash,omitempty"` // H256 (32 bytes) - block hash (hex) + BytecodeSize int `json:"bytecode_size,omitempty"` // Size of contract bytecode + BytecodeRaw string `json:"bytecode_raw,omitempty"` // First 32 bytes of raw bytecode (hex) +} + +// EVMEventInfo represents a decoded EVM Log event. +// See: _oasis-sdk/runtime-sdk/modules/evm/src/lib.rs:253-263 (Event::Log) +// See: _oasis-sdk/client-sdk/go/modules/evm/types.go:56-61 (Event) +type EVMEventInfo struct { + Address string `json:"address"` // H160 contract address (hex, 40 chars) + TopicCount int `json:"topic_count"` // Number of topics (0-4) + Topics []string `json:"topics,omitempty"` // H256 topics array (hex, 64 chars each) + EventSignature string `json:"event_signature,omitempty"` // Human-readable signature (if known) + EventHash string `json:"event_hash,omitempty"` // topic[0] keccak256 hash + DataSize int `json:"data_size"` // Size of non-indexed data + DataRaw string `json:"data_raw,omitempty"` // First 64 bytes of raw data (hex) + DecodeError string `json:"decode_error,omitempty"` +} + +// EVMTxInputInfo represents decoded EVM transaction input artifacts. +// See: _oasis-core/go/runtime/transaction/transaction.go:126-140 (inputArtifacts) +type EVMTxInputInfo struct { + TxHash string `json:"tx_hash"` // Transaction hash (hex, 64 chars) + BatchOrder uint32 `json:"batch_order"` // Order within batch + InputSize int `json:"input_size"` // Raw input size + Method string `json:"method,omitempty"` // SDK method (e.g., "evm.Call") + EVMCall *EVMCallInfo `json:"evm_call,omitempty"` // EVM-specific call details + DecodeError string `json:"decode_error,omitempty"` +} + +// EVMCallInfo represents EVM-specific transaction call details. +// See: _oasis-sdk/runtime-sdk/modules/evm/src/types.rs:10-16 (Call/Create) +type EVMCallInfo struct { + Type string `json:"type"` // "call" or "create" + Address string `json:"address,omitempty"` // H160 target address (hex, 40 chars) + Value string `json:"value,omitempty"` // U256 value in wei (decimal string) + DataSize int `json:"data_size"` // Size of call data or init code + DataRaw string `json:"data_raw,omitempty"` // First 32 bytes of raw data (hex) +} + +// EVMTxOutputInfo represents decoded EVM transaction output artifacts. +// See: _oasis-core/go/runtime/transaction/transaction.go:142-150 (outputArtifacts) +type EVMTxOutputInfo struct { + OutputSize int `json:"output_size"` // Raw output size + Success bool `json:"success"` // Transaction succeeded + ResultSize int `json:"result_size,omitempty"` // Size of result data + ResultRaw string `json:"result_raw,omitempty"` // First 32 bytes of raw result (hex) + Error string `json:"error,omitempty"` // Error message if failed + DecodeError string `json:"decode_error,omitempty"` +} diff --git a/badgerdb-sampler/types_runtime.go b/badgerdb-sampler/types_runtime.go index 8864269..a6862a3 100644 --- a/badgerdb-sampler/types_runtime.go +++ b/badgerdb-sampler/types_runtime.go @@ -172,24 +172,14 @@ type RuntimeHistoryRoundResultsInfo struct { // RuntimeLeafValueInfo represents a decoded MKVS leaf node value. // See: _oasis-core/go/runtime/transaction/transaction.go:129-150 (artifacts) type RuntimeLeafValueInfo struct { - ValueType string `json:"value_type"` // "io_input", "io_output", "io_event", "evm_code", "evm_storage", "evm_block_hash", "cbor", "binary" - InputSize int `json:"input_size,omitempty"` - BatchOrder uint32 `json:"batch_order,omitempty"` - OutputSize int `json:"output_size,omitempty"` - DecodedValue interface{} `json:"decoded_value,omitempty"` // For CBOR/event values - BinarySize int `json:"binary_size,omitempty"` // For binary values - EVM *EVMDataInfo `json:"evm,omitempty"` // For EVM-specific data -} - -// EVMDataInfo represents decoded EVM storage data. -// See: _oasis-sdk/runtime-sdk/modules/evm/src/state.rs -type EVMDataInfo struct { - StorageType string `json:"storage_type"` // "code", "storage", "block_hash", "confidential_storage" - Address string `json:"address,omitempty"` // H160 (20 bytes) - contract address (hex) - StorageSlot string `json:"storage_slot,omitempty"` // H256 (32 bytes) - storage slot (hex) - StorageValue string `json:"storage_value,omitempty"` // H256 (32 bytes) - storage value (hex) - Round uint64 `json:"round,omitempty"` // For block_hash type - BlockHash string `json:"block_hash,omitempty"` // H256 (32 bytes) - block hash (hex) - CodeSize int `json:"code_size,omitempty"` // Size of contract bytecode - CodePreview string `json:"code_preview,omitempty"` // First few bytes of bytecode (hex) + ValueType string `json:"value_type"` // "io_input", "io_output", "io_event", "evm_event", "evm_code", "evm_storage", "evm_block_hash", "cbor", "binary" + InputSize int `json:"input_size,omitempty"` + BatchOrder uint32 `json:"batch_order,omitempty"` + OutputSize int `json:"output_size,omitempty"` + DecodedValue interface{} `json:"decoded_value,omitempty"` // For CBOR/event values + BinarySize int `json:"binary_size,omitempty"` // For binary values + EVM *EVMDataInfo `json:"evm,omitempty"` // For EVM-specific data + EVMEvent *EVMEventInfo `json:"evm_event,omitempty"` // For EVM event data + EVMTxInput *EVMTxInputInfo `json:"evm_tx_input,omitempty"` // For EVM transaction input + EVMTxOutput *EVMTxOutputInfo `json:"evm_tx_output,omitempty"` // For EVM transaction output } diff --git a/badgerdb-sampler/utils.go b/badgerdb-sampler/utils.go index 6316f58..d3a9d2f 100644 --- a/badgerdb-sampler/utils.go +++ b/badgerdb-sampler/utils.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "math/big" "strings" ) @@ -201,3 +202,12 @@ func truncateHex(data []byte, maxLen int) string { return hex } +// formatU256 converts big-endian U256 bytes to decimal string. +func formatU256(b []byte) string { + if len(b) == 0 { + return "0" + } + n := new(big.Int).SetBytes(b) + return n.String() +} + From 2d75cdb5e6f8f80d34bb8c89b7aa16f524bb0a62 Mon Sep 17 00:00:00 2001 From: gw0 Date: Tue, 25 Nov 2025 10:41:34 +0100 Subject: [PATCH 13/22] badgerdb-sampler: Add decoding of consensus transactions and events --- badgerdb-sampler/decode_consensus.go | 366 +++++++++++++++--- badgerdb-sampler/decode_evm.go | 48 +-- badgerdb-sampler/decode_runtime.go | 41 +- badgerdb-sampler/types_consensus.go | 538 +++++++++++++++++++++++++-- badgerdb-sampler/types_evm.go | 46 +-- badgerdb-sampler/types_runtime.go | 35 +- badgerdb-sampler/utils.go | 118 +++++- 7 files changed, 1016 insertions(+), 176 deletions(-) diff --git a/badgerdb-sampler/decode_consensus.go b/badgerdb-sampler/decode_consensus.go index f47fbf8..31b784b 100644 --- a/badgerdb-sampler/decode_consensus.go +++ b/badgerdb-sampler/decode_consensus.go @@ -9,8 +9,6 @@ import ( "github.com/fxamacker/cbor/v2" "github.com/gogo/protobuf/proto" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" - tmstore "github.com/tendermint/tendermint/proto/tendermint/store" ) // decodeKeyConsensusBlockstore parses consensus-blockstore key and returns structured info. @@ -43,7 +41,7 @@ func decodeKeyConsensusBlockstore(key []byte) ConsensusBlockstoreKeyInfo { if strings.HasPrefix(asciiKey, "H:") { info.KeyType = "block_meta" if h, err := strconv.ParseInt(asciiKey[2:], 10, 64); err == nil { - info.Height = h + info.ConsensusHeight = h } return info } @@ -53,7 +51,7 @@ func decodeKeyConsensusBlockstore(key []byte) ConsensusBlockstoreKeyInfo { parts := strings.Split(asciiKey[2:], ":") if len(parts) >= 1 { if h, err := strconv.ParseInt(parts[0], 10, 64); err == nil { - info.Height = h + info.ConsensusHeight = h } } if len(parts) >= 2 { @@ -66,14 +64,14 @@ func decodeKeyConsensusBlockstore(key []byte) ConsensusBlockstoreKeyInfo { if strings.HasPrefix(asciiKey, "C:") { info.KeyType = "block_commit" if h, err := strconv.ParseInt(asciiKey[2:], 10, 64); err == nil { - info.Height = h + info.ConsensusHeight = h } return info } if strings.HasPrefix(asciiKey, "SC:") { info.KeyType = "seen_commit" if h, err := strconv.ParseInt(asciiKey[3:], 10, 64); err == nil { - info.Height = h + info.ConsensusHeight = h } return info } @@ -97,22 +95,22 @@ func decodeValueConsensusBlockstore(keyType string, value []byte) ConsensusBlock switch keyType { case "blockstore_state": - var state tmstore.BlockStoreState + var state tmBlockStoreState if err := proto.Unmarshal(value, &state); err == nil { info.State = &ConsensusBlockStoreState{ - Base: state.Base, - Height: state.Height, + Base: state.Base, + ConsensusHeight: state.Height, } } else { info.DecodeError = err.Error() } case "block_meta": - var meta tmproto.BlockMeta + var meta tmBlockMeta if err := proto.Unmarshal(value, &meta); err == nil { blockMeta := &ConsensusBlockMetaInfo{ - Height: meta.Header.Height, - NumTxs: meta.NumTxs, + ConsensusHeight: meta.Header.Height, + NumTxs: meta.NumTxs, } // Extract timestamp @@ -121,12 +119,8 @@ func decodeValueConsensusBlockstore(keyType string, value []byte) ConsensusBlock blockMeta.Time = meta.Header.Time.Format(time.RFC3339) } - // Format AppHash - if len(meta.Header.AppHash) >= 8 { - blockMeta.AppHash = fmt.Sprintf("%x", meta.Header.AppHash[:8]) - } else if len(meta.Header.AppHash) > 0 { - blockMeta.AppHash = fmt.Sprintf("%x", meta.Header.AppHash) - } + // Format AppHash (truncate to 16 bytes = 32 hex chars) + blockMeta.AppHash = truncateHex(meta.Header.AppHash, 32) // Truncate ChainID chainID := meta.Header.ChainID @@ -141,7 +135,7 @@ func decodeValueConsensusBlockstore(keyType string, value []byte) ConsensusBlock } case "block_part": - var part tmproto.Part + var part tmPart if err := proto.Unmarshal(value, &part); err == nil { info.Part = &ConsensusPartInfo{ Index: part.Index, @@ -153,12 +147,12 @@ func decodeValueConsensusBlockstore(keyType string, value []byte) ConsensusBlock } case "block_commit", "seen_commit": - var commit tmproto.Commit + var commit tmCommit if err := proto.Unmarshal(value, &commit); err == nil { info.Commit = &ConsensusCommitInfo{ - Height: commit.Height, - Round: commit.Round, - Signatures: len(commit.Signatures), + ConsensusHeight: commit.Height, + Round: commit.Round, + Signatures: len(commit.Signatures), } } else { info.DecodeError = err.Error() @@ -213,24 +207,43 @@ func decodeValueConsensusEvidence(keyType string, value []byte) ConsensusEvidenc return info } - // Try to decode as DuplicateVoteEvidence - var evidence tmproto.DuplicateVoteEvidence - if err := proto.Unmarshal(value, &evidence); err == nil { - if evidence.VoteA != nil { - info.VoteAHeight = evidence.VoteA.Height + // Try to decode as DuplicateVoteEvidence first + var dve tmDuplicateVoteEvidence + if err := proto.Unmarshal(value, &dve); err == nil && dve.VoteA != nil { + info.EvidenceType = "duplicate_vote" + info.VoteAHeight = dve.VoteA.Height + if dve.VoteB != nil { + info.VoteBHeight = dve.VoteB.Height + } + info.TotalVotingPower = dve.TotalVotingPower + info.ValidatorPower = dve.ValidatorPower + if dve.Timestamp.Unix() > 0 { + info.Timestamp = dve.Timestamp.Format(time.RFC3339) } - if evidence.VoteB != nil { - info.VoteBHeight = evidence.VoteB.Height + return info + } + + // Try to decode as LightClientAttackEvidence + var lca tmLightClientAttackEvidence + if err := proto.Unmarshal(value, &lca); err == nil && lca.ConflictingBlock != nil { + info.EvidenceType = "light_client_attack" + info.TotalVotingPower = lca.TotalVotingPower + if lca.Timestamp.Unix() > 0 { + info.Timestamp = lca.Timestamp.Format(time.RFC3339) } - } else { - info.DecodeError = err.Error() + return info } + // If all parsing failed, show raw value + info.EvidenceType = "unknown" + info.RawValue = truncateHex(value, 64) + info.DecodeError = "unable to decode evidence format" + return info } // decodeKeyConsensusMkvs parses consensus-mkvs key and returns structured info. -// Handles both old format (no dbVersion prefix) and new format (0x01 or 0x05 dbVersion prefix). +// Keys use keyformat encoding: [type_byte][data...] // See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 func decodeKeyConsensusMkvs(key []byte) ConsensusMkvsKeyInfo { info := ConsensusMkvsKeyInfo{} @@ -241,22 +254,9 @@ func decodeKeyConsensusMkvs(key []byte) ConsensusMkvsKeyInfo { return info } - // Check if key has dbVersion prefix (0x01 or 0x05) + // First byte is the key type prefix prefixByte := key[0] - var data []byte - - if prefixByte == 0x01 || prefixByte == 0x05 { - info.DbPrefix = prefixByte - if len(key) < 2 { - info.KeyType = "unknown" - info.DecodeError = "key too short after prefix" - return info - } - prefixByte = key[1] - data = key[2:] - } else { - data = key[1:] - } + data := key[1:] switch prefixByte { case 0x00: @@ -266,10 +266,10 @@ func decodeKeyConsensusMkvs(key []byte) ConsensusMkvsKeyInfo { case 0x01: info.KeyType = "write_log" if len(data) >= 8 { - info.Height = binary.BigEndian.Uint64(data[0:8]) + info.ConsensusHeight = binary.BigEndian.Uint64(data[0:8]) if len(data) >= 8+33 { info.RootType = data[8] - info.Hash = truncateHex(data[9:9+32], 8) + info.Hash = truncateHex(data[9:9+32], 32) } } else { info.DecodeError = "write_log data too short" @@ -278,7 +278,7 @@ func decodeKeyConsensusMkvs(key []byte) ConsensusMkvsKeyInfo { case 0x02: info.KeyType = "roots_metadata" if len(data) >= 8 { - info.Height = binary.BigEndian.Uint64(data[0:8]) + info.ConsensusHeight = binary.BigEndian.Uint64(data[0:8]) } else { info.DecodeError = "roots_metadata data too short" } @@ -286,10 +286,10 @@ func decodeKeyConsensusMkvs(key []byte) ConsensusMkvsKeyInfo { case 0x03: info.KeyType = "root_updated_nodes" if len(data) >= 8 { - info.Height = binary.BigEndian.Uint64(data[0:8]) + info.ConsensusHeight = binary.BigEndian.Uint64(data[0:8]) if len(data) >= 8+33 { info.RootType = data[8] - info.Hash = truncateHex(data[9:9+32], 8) + info.Hash = truncateHex(data[9:9+32], 32) } } else { info.DecodeError = "root_updated_nodes data too short" @@ -370,6 +370,22 @@ func decodeValueConsensusMkvs(keyType string, value []byte) ConsensusMkvsNodeInf Key: truncateHex(key, 32), } + // Extract Oasis address for staking-related modules + // Staking keys have format: module_prefix (1 byte) + 21-byte Oasis address + if keyLen == 22 { + switch key[0] { + case 0x34, 0x35, 0x36, 0x37: // staking accounts, delegations, debonding, allowances + leaf.OasisAddress = bech32Encode("oasis", key[1:22]) + } + } + // Entity/node keys also contain addresses + if keyLen >= 22 { + switch key[0] { + case 0x40, 0x41: // registry entities, nodes + leaf.OasisAddress = bech32Encode("oasis", key[1:22]) + } + } + if len(data) < 4 { info.DecodeError = "value length missing" info.Leaf = leaf @@ -587,10 +603,28 @@ func decodeKeyConsensusState(key []byte) ConsensusStateKeyInfo { switch prefix { case "abciResponsesKey": info.KeyType = "abci_responses" + // Extract height from key like "abciResponsesKey:10000000" + if colonIdx < len(decoded)-1 { + if h, err := strconv.ParseInt(decoded[colonIdx+1:], 10, 64); err == nil { + info.ConsensusHeight = h + } + } case "consensusParamsKey": info.KeyType = "consensus_params" + // Extract height if present + if colonIdx < len(decoded)-1 { + if h, err := strconv.ParseInt(decoded[colonIdx+1:], 10, 64); err == nil { + info.ConsensusHeight = h + } + } case "validatorsKey": info.KeyType = "validators" + // Extract height if present + if colonIdx < len(decoded)-1 { + if h, err := strconv.ParseInt(decoded[colonIdx+1:], 10, 64); err == nil { + info.ConsensusHeight = h + } + } case "stateKey": info.KeyType = "state" case "genesisDoc": @@ -607,8 +641,232 @@ func decodeKeyConsensusState(key []byte) ConsensusStateKeyInfo { // decodeValueConsensusState decodes state value and returns structured info. func decodeValueConsensusState(keyType string, value []byte) ConsensusStateValueInfo { - return ConsensusStateValueInfo{ + info := ConsensusStateValueInfo{ KeyType: keyType, Size: len(value), } + + if len(value) == 0 { + return info + } + + switch keyType { + case "abci_responses": + // Try to decode ABCI responses (ResponseFinalizeBlock in newer Tendermint) + var resp tmABCIResponses + if err := proto.Unmarshal(value, &resp); err == nil { + abciInfo := &ABCIResponseInfo{} + + // Count deliver_tx results (transaction results) + if resp.DeliverTxs != nil { + abciInfo.TxResultCount = len(resp.DeliverTxs) + } + + // Count events from end_block + if resp.EndBlock != nil { + abciInfo.EventCount = len(resp.EndBlock.Events) + if resp.EndBlock.ValidatorUpdates != nil { + abciInfo.ValidatorUpdates = len(resp.EndBlock.ValidatorUpdates) + } + } + + // Count event types from all sources + eventSummary := &ConsensusEventSummary{ + EventTypeCounts: make(map[string]int), + } + + // Decode transactions and collect events from DeliverTxs + txSummary := &ConsensusTransactionSummary{ + MethodCounts: make(map[string]int), + } + + if resp.DeliverTxs != nil { + for _, txResult := range resp.DeliverTxs { + // Count events + for _, event := range txResult.Events { + eventSummary.EventTypeCounts[event.Type]++ + } + + // Decode transaction if present + if len(txResult.Data) > 0 { + decodedTx := decodeCBORTransaction(txResult.Data) + if decodedTx.DecodeError == "" { + txSummary.MethodCounts[decodedTx.Method]++ + // Only include first few transactions for sample + if len(txSummary.Transactions) < 5 { + txSummary.Transactions = append(txSummary.Transactions, decodedTx) + } + } + } + } + } + + if len(txSummary.MethodCounts) > 0 { + abciInfo.TransactionSummary = txSummary + } + + // Events from BeginBlock + if resp.BeginBlock != nil { + for _, event := range resp.BeginBlock.Events { + eventSummary.EventTypeCounts[event.Type]++ + } + } + + // Events from EndBlock + if resp.EndBlock != nil { + for _, event := range resp.EndBlock.Events { + eventSummary.EventTypeCounts[event.Type]++ + } + } + + if len(eventSummary.EventTypeCounts) > 0 { + abciInfo.EventSummary = eventSummary + } + + info.ABCIInfo = abciInfo + } else { + info.DecodeError = fmt.Sprintf("failed to decode ABCI responses: %v", err) + } + + case "consensus_params": + info.RawValue = truncateHex(value, 32) + var params tmConsensusParams + if err := proto.Unmarshal(value, ¶ms); err == nil { + info.ConsensusParams = ¶ms + } else { + info.DecodeError = fmt.Sprintf("failed to decode consensus params: %v", err) + } + + case "validators": + info.RawValue = truncateHex(value, 32) + var valSet tmValidatorSet + if err := proto.Unmarshal(value, &valSet); err == nil { + info.ConsensusValidators = &valSet + } else { + info.DecodeError = fmt.Sprintf("failed to decode validator set: %v", err) + } + + case "state": + info.RawValue = truncateHex(value, 32) + var state tmState + if err := proto.Unmarshal(value, &state); err == nil { + info.ConsensusState = &state + } else { + info.DecodeError = fmt.Sprintf("failed to decode state: %v", err) + } + + case "genesis": + info.RawValue = truncateHex(value, 64) + if value[0] != '{' { + info.DecodeError = "genesis document not in expected JSON format" + } + // Genesis is JSON, RawValue shows hex preview + + default: + info.RawValue = truncateHex(value, 32) + } + + return info +} + +// decodeCBORTransaction decodes a raw CBOR-encoded transaction bytes into ConsensusDecodedTransaction. +// See: _oasis-core/go/consensus/api/transaction/transaction.go:42-54 +// See: _oasis-core/go/common/crypto/signature/signature.go:415-421 +// See: _oasis-core/go/staking/api/api.go for transaction body types +func decodeCBORTransaction(rawTx []byte) ConsensusDecodedTransaction { + decoded := ConsensusDecodedTransaction{} + + // Decode SignedTransaction envelope + var signedTx ConsensusSignedTransaction + if err := cbor.Unmarshal(rawTx, &signedTx); err != nil { + decoded.DecodeError = fmt.Sprintf("failed to decode SignedTransaction: %v", err) + return decoded + } + + // Extract signer public key + decoded.Signer = fmt.Sprintf("%x", signedTx.Signature.PublicKey[:]) + + // Decode inner Transaction from the blob + var tx struct { + Nonce uint64 `cbor:"nonce"` + Fee *ConsensusFee `cbor:"fee"` + Method string `cbor:"method"` + Body cbor.RawMessage `cbor:"body"` + } + if err := cbor.Unmarshal(signedTx.Blob, &tx); err != nil { + decoded.DecodeError = fmt.Sprintf("failed to decode Transaction: %v", err) + return decoded + } + + decoded.Nonce = tx.Nonce + decoded.Method = tx.Method + decoded.Fee = tx.Fee + + // Decode body based on method + if len(tx.Body) > 0 { + decoded.BodyPreview = truncateHex(tx.Body, 32) + + switch tx.Method { + case "staking.Transfer": + var transfer ConsensusTransfer + if err := cbor.Unmarshal(tx.Body, &transfer); err == nil { + transfer.Amount = quantityBytesToString(tx.Body, "amount") + decoded.DecodedBody = transfer + } + + case "staking.Burn": + var burn ConsensusBurn + if err := cbor.Unmarshal(tx.Body, &burn); err == nil { + burn.Amount = quantityBytesToString(tx.Body, "amount") + decoded.DecodedBody = burn + } + + case "staking.AddEscrow": + var escrow ConsensusAddEscrow + if err := cbor.Unmarshal(tx.Body, &escrow); err == nil { + escrow.Amount = quantityBytesToString(tx.Body, "amount") + decoded.DecodedBody = escrow + } + + case "staking.ReclaimEscrow": + var reclaim ConsensusReclaimEscrow + if err := cbor.Unmarshal(tx.Body, &reclaim); err == nil { + reclaim.Shares = quantityBytesToString(tx.Body, "shares") + decoded.DecodedBody = reclaim + } + + case "registry.RegisterEntity": + var regEntity ConsensusRegisterEntity + if err := cbor.Unmarshal(tx.Body, ®Entity); err == nil { + decoded.DecodedBody = regEntity + } + + case "registry.RegisterNode": + var regNode ConsensusRegisterNode + if err := cbor.Unmarshal(tx.Body, ®Node); err == nil { + decoded.DecodedBody = regNode + } + + case "roothash.ExecutorCommit": + var execCommit ConsensusExecutorCommit + if err := cbor.Unmarshal(tx.Body, &execCommit); err == nil { + decoded.DecodedBody = execCommit + } + + case "governance.SubmitProposal": + var proposal ConsensusSubmitProposal + if err := cbor.Unmarshal(tx.Body, &proposal); err == nil { + proposal.Deposit = quantityBytesToString(tx.Body, "deposit") + decoded.DecodedBody = proposal + } + + case "governance.CastVote": + var vote ConsensusCastVote + if err := cbor.Unmarshal(tx.Body, &vote); err == nil { + decoded.DecodedBody = vote + } + } + } + + return decoded } diff --git a/badgerdb-sampler/decode_evm.go b/badgerdb-sampler/decode_evm.go index 235582f..e01afeb 100644 --- a/badgerdb-sampler/decode_evm.go +++ b/badgerdb-sampler/decode_evm.go @@ -41,7 +41,7 @@ func decodeEVMData(module string, key []byte, value []byte) *EVMDataInfo { case 0x01: // CODES: evm + 0x01 + H160 (address) evmInfo.StorageType = "code" if len(data) >= 20 { - evmInfo.Address = fmt.Sprintf("%x", data[:20]) + evmInfo.Address = fmt.Sprintf("0x%x", data[:20]) data = data[20:] } // Value is contract bytecode @@ -51,40 +51,40 @@ func decodeEVMData(module string, key []byte, value []byte) *EVMDataInfo { if len(value) < previewLen { previewLen = len(value) } - evmInfo.BytecodeRaw = fmt.Sprintf("%x", value[:previewLen]) + evmInfo.BytecodeRaw = fmt.Sprintf("0x%x", value[:previewLen]) } case 0x02: // STORAGES: evm + 0x02 + H160 (address) + H256 (slot) evmInfo.StorageType = "storage" if len(data) >= 20 { - evmInfo.Address = fmt.Sprintf("%x", data[:20]) + evmInfo.Address = fmt.Sprintf("0x%x", data[:20]) data = data[20:] if len(data) >= 32 { - evmInfo.StorageSlot = fmt.Sprintf("%x", data[:32]) + evmInfo.StorageSlot = fmt.Sprintf("0x%x", data[:32]) } } // Value is H256 storage value if len(value) == 32 { - evmInfo.StorageValue = fmt.Sprintf("%x", value) + evmInfo.StorageValue = fmt.Sprintf("0x%x", value) } - case 0x03: // BLOCK_HASHES: evm + 0x03 + Round (uint64 BE) + case 0x03: // BLOCK_HASHES: evm + 0x03 + RuntimeHeight (uint64 BE) evmInfo.StorageType = "block_hash" if len(data) >= 8 { - evmInfo.Round = binary.BigEndian.Uint64(data[:8]) + evmInfo.RuntimeHeight = binary.BigEndian.Uint64(data[:8]) } // Value is H256 block hash if len(value) == 32 { - evmInfo.BlockHash = fmt.Sprintf("%x", value) + evmInfo.BlockHash = fmt.Sprintf("0x%x", value) } case 0x04: // CONFIDENTIAL_STORAGES: evm + 0x04 + H160 (address) + H256 (slot) evmInfo.StorageType = "confidential_storage" if len(data) >= 20 { - evmInfo.Address = fmt.Sprintf("%x", data[:20]) + evmInfo.Address = fmt.Sprintf("0x%x", data[:20]) data = data[20:] if len(data) >= 32 { - evmInfo.StorageSlot = fmt.Sprintf("%x", data[:32]) + evmInfo.StorageSlot = fmt.Sprintf("0x%x", data[:32]) } } // Value is encrypted - we can only show size @@ -123,7 +123,7 @@ func decodeEVMEvent(value []byte) *EVMEventInfo { // Extract address (H160) if addrBytes, ok := eventData["address"].([]byte); ok && len(addrBytes) == 20 { - info.Address = fmt.Sprintf("%x", addrBytes) + info.Address = fmt.Sprintf("0x%x", addrBytes) } else { // Continue decoding even with invalid address - topics and data may still be useful info.DecodeError = "invalid address (continuing decode)" @@ -135,9 +135,9 @@ func decodeEVMEvent(value []byte) *EVMEventInfo { for i, topic := range topicsArray { if topicBytes, ok := topic.([]byte); ok && len(topicBytes) == 32 { topicHex := fmt.Sprintf("%x", topicBytes) - info.Topics = append(info.Topics, topicHex) + info.Topics = append(info.Topics, "0x"+topicHex) if i == 0 { - info.EventHash = topicHex + info.EventHash = "0x" + topicHex if sig, found := EVMEventSignatures[topicHex]; found { info.EventSignature = sig } @@ -149,7 +149,7 @@ func decodeEVMEvent(value []byte) *EVMEventInfo { // Extract data if dataBytes, ok := eventData["data"].([]byte); ok { info.DataSize = len(dataBytes) - info.DataRaw = truncateHex(dataBytes, 128) // 64 bytes = 128 hex chars + info.DataRaw = truncateHex0x(dataBytes, 128) // 64 bytes = 128 hex chars } return info @@ -160,7 +160,7 @@ func decodeEVMTxInput(key []byte, value []byte) *EVMTxInputInfo { info := &EVMTxInputInfo{InputSize: len(value)} if len(key) >= 33 { - info.TxHash = fmt.Sprintf("%x", key[1:33]) + info.TxHash = fmt.Sprintf("0x%x", key[1:33]) } var ia RuntimeInputArtifacts @@ -174,10 +174,16 @@ func decodeEVMTxInput(key []byte, value []byte) *EVMTxInputInfo { if err := cbor.Unmarshal(ia.Input, &call); err == nil { if methodVal, ok := call["method"].(string); ok { info.Method = methodVal - if methodVal == "evm.Call" || methodVal == "evm.Create" { - info.EVMCall = decodeEVMCallBody(call, methodVal == "evm.Create") + // Decode EVM call methods that have body structure + if methodVal == "evm.Call" || methodVal == "evm.Create" || methodVal == "evm.SimulateCall" || methodVal == "evm.EstimateGas" { + isCreate := methodVal == "evm.Create" + info.EVMCall = decodeEVMCallBody(call, isCreate) } + } else { + info.DecodeError = "method field missing or invalid in call structure" } + } else { + info.DecodeError = fmt.Sprintf("failed to unmarshal call structure: %v", err) } return info @@ -205,7 +211,7 @@ func decodeEVMCallBody(call map[interface{}]interface{}, isCreate bool) *EVMCall // Extract address (calls only) if !isCreate { if addrBytes, ok := bodyMap["address"].([]byte); ok && len(addrBytes) == 20 { - info.Address = fmt.Sprintf("%x", addrBytes) + info.Address = fmt.Sprintf("0x%x", addrBytes) } } @@ -221,7 +227,7 @@ func decodeEVMCallBody(call map[interface{}]interface{}, isCreate bool) *EVMCall } if dataBytes, ok := bodyMap[dataKey].([]byte); ok { info.DataSize = len(dataBytes) - info.DataRaw = truncateHex(dataBytes, 64) // 32 bytes = 64 hex chars + info.DataRaw = truncateHex0x(dataBytes, 64) // 32 bytes = 64 hex chars } return info @@ -248,7 +254,7 @@ func decodeEVMTxOutput(value []byte) *EVMTxOutputInfo { // Raw bytes - success info.Success = true info.ResultSize = len(oa.Output) - info.ResultRaw = truncateHex(oa.Output, 64) // 32 bytes = 64 hex chars + info.ResultRaw = truncateHex0x(oa.Output, 64) // 32 bytes = 64 hex chars return info } @@ -257,7 +263,7 @@ func decodeEVMTxOutput(value []byte) *EVMTxOutputInfo { info.Success = true if okBytes, ok := okVal.([]byte); ok { info.ResultSize = len(okBytes) - info.ResultRaw = truncateHex(okBytes, 64) + info.ResultRaw = truncateHex0x(okBytes, 64) } } else if failVal, exists := result["fail"]; exists { info.Success = false diff --git a/badgerdb-sampler/decode_runtime.go b/badgerdb-sampler/decode_runtime.go index cdc5c56..3fc062a 100644 --- a/badgerdb-sampler/decode_runtime.go +++ b/badgerdb-sampler/decode_runtime.go @@ -9,7 +9,7 @@ import ( ) // decodeKeyRuntimeMkvs parses runtime-mkvs key and returns structured info. -// Key format: [optional db prefix 0x01|0x05][type byte][data...] +// Keys use keyformat encoding: [type_byte][data...] // See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 func decodeKeyRuntimeMkvs(key []byte) RuntimeMkvsKeyInfo { info := RuntimeMkvsKeyInfo{} @@ -20,22 +20,9 @@ func decodeKeyRuntimeMkvs(key []byte) RuntimeMkvsKeyInfo { return info } - // Check for dbVersion prefix (0x01 or 0x05) + // First byte is the key type prefix prefixByte := key[0] - var data []byte - - if prefixByte == 0x01 || prefixByte == 0x05 { - info.DbPrefix = prefixByte - if len(key) < 2 { - info.KeyType = "unknown" - info.DecodeError = "key too short after prefix" - return info - } - prefixByte = key[1] - data = key[2:] - } else { - data = key[1:] - } + data := key[1:] switch prefixByte { case 0x00: @@ -44,21 +31,21 @@ func decodeKeyRuntimeMkvs(key []byte) RuntimeMkvsKeyInfo { case 0x01: info.KeyType = "write_log" if len(data) >= 8 { - info.Height = binary.BigEndian.Uint64(data[0:8]) + info.RuntimeHeight = binary.BigEndian.Uint64(data[0:8]) } else { info.DecodeError = "write_log data too short" } case 0x02: info.KeyType = "roots_metadata" if len(data) >= 8 { - info.Height = binary.BigEndian.Uint64(data[0:8]) + info.RuntimeHeight = binary.BigEndian.Uint64(data[0:8]) } else { info.DecodeError = "roots_metadata data too short" } case 0x03: info.KeyType = "root_updated_nodes" if len(data) >= 8 { - info.Height = binary.BigEndian.Uint64(data[0:8]) + info.RuntimeHeight = binary.BigEndian.Uint64(data[0:8]) } else { info.DecodeError = "root_updated_nodes data too short" } @@ -215,7 +202,7 @@ func decodeKeyRuntimeHistory(key []byte) RuntimeHistoryKeyInfo { case 0x02: info.KeyType = "block" if len(key) == 9 { - info.Height = binary.BigEndian.Uint64(key[1:9]) + info.RuntimeHeight = binary.BigEndian.Uint64(key[1:9]) } else { info.DecodeError = "block key wrong length" } @@ -223,7 +210,7 @@ func decodeKeyRuntimeHistory(key []byte) RuntimeHistoryKeyInfo { case 0x03: info.KeyType = "round_results" if len(key) == 9 { - info.Height = binary.BigEndian.Uint64(key[1:9]) + info.RuntimeHeight = binary.BigEndian.Uint64(key[1:9]) } else { info.DecodeError = "round_results key wrong length" } @@ -259,10 +246,10 @@ func decodeValueRuntimeHistory(keyType string, value []byte) RuntimeHistoryValue return info } info.Metadata = &RuntimeHistoryMetadataInfo{ - Version: meta.Version, - RuntimeID: fmt.Sprintf("%x", meta.RuntimeID), - LastRound: meta.LastRound, - LastConsensusHeight: meta.LastConsensusHeight, + Version: meta.Version, + RuntimeID: fmt.Sprintf("%x", meta.RuntimeID), + LastRuntimeHeight: meta.LastRound, + LastConsensusHeight: meta.LastConsensusHeight, } case "block": @@ -278,10 +265,10 @@ func decodeValueRuntimeHistory(keyType string, value []byte) RuntimeHistoryValue blockInfo.BlockNil = true } else { h := block.Block.Header - blockInfo.Round = h.Round + blockInfo.RuntimeHeight = h.Round blockInfo.Timestamp = time.Unix(int64(h.Timestamp), 0).UTC().Format(time.RFC3339) blockInfo.HeaderType = headerTypeName(h.HeaderType) - blockInfo.StateRoot = truncateHex(h.StateRoot, 16) + blockInfo.StateRoot = truncateHex(h.StateRoot, 32) } info.Block = blockInfo diff --git a/badgerdb-sampler/types_consensus.go b/badgerdb-sampler/types_consensus.go index 5b9047f..fcf66b6 100644 --- a/badgerdb-sampler/types_consensus.go +++ b/badgerdb-sampler/types_consensus.go @@ -1,5 +1,7 @@ package main +import "time" + // Consensus decode output types - structured representations of decoded consensus database entries. // These types separate decoding logic from string formatting, enabling flexible output formats. @@ -10,11 +12,11 @@ package main // ConsensusBlockstoreKeyInfo represents a decoded consensus-blockstore key. // See: tendermint/store/store.go for key formats (H:, P:, C:, SC:, BH:) type ConsensusBlockstoreKeyInfo struct { - KeyType string `json:"key_type"` // "blockstore_state", "block_meta", "block_part", "block_commit", "seen_commit", "block_hash", "unknown" - Height int64 `json:"height,omitempty"` // For H:, C:, SC:, P: keys - PartIndex int `json:"part_index,omitempty"` // For P: keys - Hash string `json:"hash,omitempty"` // For BH: keys - RawKey string `json:"raw_key,omitempty"` // Original ASCII key + KeyType string `json:"key_type"` // "blockstore_state", "block_meta", "block_part", "block_commit", "seen_commit", "block_hash", "unknown" + ConsensusHeight int64 `json:"consensus_height,omitempty"` // For H:, C:, SC:, P: keys + PartIndex int `json:"part_index,omitempty"` // For P: keys + Hash string `json:"hash,omitempty"` // For BH: keys + RawKey string `json:"raw_key,omitempty"` // Original ASCII key } // ConsensusBlockstoreValueInfo represents a decoded consensus-blockstore value. @@ -35,18 +37,18 @@ type ConsensusBlockstoreValueInfo struct { // ConsensusBlockStoreState represents BlockStoreState from tendermint. // See: tendermint/proto/tendermint/store/types.proto (BlockStoreState message) type ConsensusBlockStoreState struct { - Base int64 `json:"base"` - Height int64 `json:"height"` + Base int64 `json:"base"` + ConsensusHeight int64 `json:"consensus_height"` } // ConsensusBlockMetaInfo represents decoded block metadata. // See: tendermint/proto/tendermint/types/types.proto (BlockMeta, Header messages) type ConsensusBlockMetaInfo struct { - Height int64 `json:"height"` - Time string `json:"time"` // RFC3339 format - ChainID string `json:"chain_id"` - NumTxs int64 `json:"num_txs"` - AppHash string `json:"app_hash,omitempty"` // hex, truncated + ConsensusHeight int64 `json:"consensus_height"` + Time string `json:"time"` // RFC3339 format + ChainID string `json:"chain_id"` + NumTxs int64 `json:"num_txs"` + AppHash string `json:"app_hash,omitempty"` // hex, truncated } // ConsensusPartInfo represents decoded block part. @@ -60,9 +62,9 @@ type ConsensusPartInfo struct { // ConsensusCommitInfo represents decoded commit. // See: tendermint/proto/tendermint/types/types.proto (Commit message) type ConsensusCommitInfo struct { - Height int64 `json:"height"` - Round int32 `json:"round"` - Signatures int `json:"signatures"` + ConsensusHeight int64 `json:"consensus_height"` + Round int32 `json:"round"` // Tendermint commit round (not runtime height) + Signatures int `json:"signatures"` } // ============================================================================= @@ -80,10 +82,15 @@ type ConsensusEvidenceKeyInfo struct { // ConsensusEvidenceValueInfo represents a decoded consensus-evidence value. // See: tendermint/proto/tendermint/types/evidence.proto (DuplicateVoteEvidence) type ConsensusEvidenceValueInfo struct { - Size int `json:"size"` - VoteAHeight int64 `json:"vote_a_height,omitempty"` - VoteBHeight int64 `json:"vote_b_height,omitempty"` - DecodeError string `json:"decode_error,omitempty"` + Size int `json:"size"` + VoteAHeight int64 `json:"vote_a_height,omitempty"` + VoteBHeight int64 `json:"vote_b_height,omitempty"` + EvidenceType string `json:"evidence_type,omitempty"` // "duplicate_vote", "light_client_attack", "unknown" + TotalVotingPower int64 `json:"total_voting_power,omitempty"` + ValidatorPower int64 `json:"validator_power,omitempty"` + Timestamp string `json:"timestamp,omitempty"` // RFC3339 format + RawValue string `json:"raw_value,omitempty"` // First 32 hex bytes if decode fails + DecodeError string `json:"decode_error,omitempty"` } // ============================================================================= @@ -93,12 +100,11 @@ type ConsensusEvidenceValueInfo struct { // ConsensusMkvsKeyInfo represents a decoded consensus-mkvs key. // See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 type ConsensusMkvsKeyInfo struct { - KeyType string `json:"key_type"` // "node", "write_log", "roots_metadata", "root_updated_nodes", "metadata", "multipart_restore_log", "root_node", "unknown" - Height uint64 `json:"height,omitempty"` // For write_log, roots_metadata, root_updated_nodes - Hash string `json:"hash,omitempty"` // hex, truncated - RootType byte `json:"root_type,omitempty"` - DbPrefix byte `json:"db_prefix,omitempty"` // 0x01 or 0x05 if present - DecodeError string `json:"decode_error,omitempty"` + KeyType string `json:"key_type"` // "node", "write_log", "roots_metadata", "root_updated_nodes", "metadata", "multipart_restore_log", "root_node", "unknown" + ConsensusHeight uint64 `json:"consensus_height,omitempty"` // For write_log, roots_metadata, root_updated_nodes + Hash string `json:"hash,omitempty"` // hex, truncated + RootType byte `json:"root_type,omitempty"` + DecodeError string `json:"decode_error,omitempty"` } // ConsensusMkvsNodeInfo represents a decoded consensus-mkvs value (node). @@ -117,7 +123,8 @@ type ConsensusMkvsNodeInfo struct { type ConsensusMkvsLeafInfo struct { Module string `json:"module"` KeyLen int `json:"key_len"` - Key string `json:"key,omitempty"` // hex, truncated + Key string `json:"key,omitempty"` // hex, truncated + OasisAddress string `json:"oasis_address,omitempty"` // bech32 oasis1... address if applicable ValueLen int `json:"value_len"` DecodedValue interface{} `json:"decoded_value,omitempty"` // CBOR decoded or size info } @@ -138,15 +145,482 @@ type ConsensusMkvsInternalInfo struct { // ConsensusStateKeyInfo represents a decoded consensus-state key. // See: tendermint/state/store.go for key formats (abciResponsesKey, validatorsKey, etc.) type ConsensusStateKeyInfo struct { - KeyType string `json:"key_type"` // "abci_responses", "consensus_params", "validators", "state", "genesis", "text_key", "binary", "unknown" - DecodedKey string `json:"decoded_key,omitempty"` - IsBinary bool `json:"is_binary,omitempty"` + KeyType string `json:"key_type"` // "abci_responses", "consensus_params", "validators", "state", "genesis", "text_key", "binary", "unknown" + DecodedKey string `json:"decoded_key,omitempty"` + ConsensusHeight int64 `json:"consensus_height,omitempty"` // For abci_responses (extracted from key) + IsBinary bool `json:"is_binary,omitempty"` } // ConsensusStateValueInfo represents a decoded consensus-state value. // See: tendermint/proto/tendermint/state/types.proto (various state types) -// Note: Tendermint state values are complex protobuf - we just report size. type ConsensusStateValueInfo struct { - KeyType string `json:"key_type"` - Size int `json:"size"` + KeyType string `json:"key_type"` + Size int `json:"size"` + ABCIInfo *ABCIResponseInfo `json:"abci_info,omitempty"` // For abci_responses + ConsensusParams *tmConsensusParams `json:"consensus_params,omitempty"` // For consensus_params + ConsensusValidators *tmValidatorSet `json:"consensus_validators,omitempty"` // For validators + ConsensusState *tmState `json:"consensus_state,omitempty"` // For state + RawValue string `json:"raw_value,omitempty"` // Raw hex dump + DecodeError string `json:"decode_error,omitempty"` +} + +// ABCIResponseInfo represents decoded ABCI response summary. +// See: tendermint/proto/tendermint/abci/types.proto (ResponseFinalizeBlock) +type ABCIResponseInfo struct { + TxResultCount int `json:"tx_result_count"` + ValidatorUpdates int `json:"validator_updates"` + EventCount int `json:"event_count"` + TransactionSummary *ConsensusTransactionSummary `json:"transaction_summary,omitempty"` + EventSummary *ConsensusEventSummary `json:"event_summary,omitempty"` +} + +// ConsensusTransactionSummary summarizes decoded transactions from ABCI responses. +type ConsensusTransactionSummary struct { + MethodCounts map[string]int `json:"method_counts"` // Count by method name + Transactions []ConsensusDecodedTransaction `json:"transactions,omitempty"` // Decoded transaction details +} + +// ConsensusEventSummary summarizes decoded events from ABCI responses. +type ConsensusEventSummary struct { + EventTypeCounts map[string]int `json:"event_type_counts"` // Count by event type +} + +// ============================================================================= +// Transaction and Event Types (Oasis-specific) +// ============================================================================= + +// ConsensusDecodedTransaction represents a decoded Oasis consensus transaction. +// See: _oasis-core/go/consensus/api/transaction/transaction.go:42-54 +type ConsensusDecodedTransaction struct { + Nonce uint64 `json:"nonce"` + Method string `json:"method"` + Fee *ConsensusFee `json:"fee,omitempty"` + Signer string `json:"signer,omitempty"` // hex-encoded public key + TxHash string `json:"tx_hash,omitempty"` // hex-encoded transaction hash + BodyPreview string `json:"body_preview,omitempty"` // hex preview if body not decoded + DecodedBody interface{} `json:"decoded_body,omitempty"` // Decoded body for known types + DecodeError string `json:"decode_error,omitempty"` +} + +// ConsensusFee represents transaction fee. +// See: _oasis-core/go/consensus/api/transaction/gas.go:30-35 +type ConsensusFee struct { + Amount string `json:"amount"` // string representation of quantity.Quantity + Gas uint64 `json:"gas"` +} + +// ConsensusSignedTransaction represents the signature envelope. +// See: _oasis-core/go/common/crypto/signature/signature.go:415-421 +type ConsensusSignedTransaction struct { + Blob []byte `json:"untrusted_raw_value"` + Signature ConsensusSignature `json:"signature"` +} + +// ConsensusSignature represents a signature with public key. +// See: _oasis-core/go/common/crypto/signature/signature.go:313-318 +type ConsensusSignature struct { + PublicKey [32]byte `json:"public_key"` // ED25519 public key + Signature [64]byte `json:"signature"` // ED25519 signature +} + +// ============================================================================= +// Transaction Body Types (Staking Module) +// ============================================================================= + +// ConsensusTransfer represents a staking transfer transaction body. +// See: _oasis-core/go/staking/api/api.go:342-345 +type ConsensusTransfer struct { + To [21]byte `json:"to"` // Oasis address + Amount string `json:"amount"` // string representation of quantity.Quantity +} + +// ConsensusBurn represents a staking burn transaction body. +// See: _oasis-core/go/staking/api/api.go:368-370 +type ConsensusBurn struct { + Amount string `json:"amount"` // string representation of quantity.Quantity +} + +// ConsensusAddEscrow represents an add escrow transaction body. +// See: _oasis-core/go/staking/api/api.go:403-406 +type ConsensusAddEscrow struct { + Account [21]byte `json:"account"` // Oasis address + Amount string `json:"amount"` // string representation of quantity.Quantity +} + +// ConsensusReclaimEscrow represents a reclaim escrow transaction body. +// See: _oasis-core/go/staking/api/api.go:444-447 +type ConsensusReclaimEscrow struct { + Account [21]byte `json:"account"` // Oasis address + Shares string `json:"shares"` // string representation of quantity.Quantity +} + +// ConsensusRegisterEntity represents a registry entity registration transaction body. +// See: _oasis-core/go/registry/api/api.go +type ConsensusRegisterEntity struct { + Signature []byte `json:"signature,omitempty"` + Descriptor []byte `json:"descriptor,omitempty"` // CBOR-encoded entity descriptor +} + +// ConsensusRegisterNode represents a registry node registration transaction body. +// See: _oasis-core/go/registry/api/api.go +type ConsensusRegisterNode struct { + Signature []byte `json:"signature,omitempty"` + Descriptor []byte `json:"descriptor,omitempty"` // CBOR-encoded node descriptor +} + +// ConsensusExecutorCommit represents a roothash executor commit transaction body. +// See: _oasis-core/go/roothash/api/commitment/executor.go +type ConsensusExecutorCommit struct { + ID []byte `json:"runtime_id,omitempty"` // Runtime ID + Commits []byte `json:"commits,omitempty"` // CBOR-encoded commits +} + +// ConsensusSubmitProposal represents a governance proposal submission transaction body. +// See: _oasis-core/go/governance/api/api.go +type ConsensusSubmitProposal struct { + Content []byte `json:"content,omitempty"` // CBOR-encoded proposal content + Deposit string `json:"deposit,omitempty"` // string representation of quantity.Quantity +} + +// ConsensusCastVote represents a governance vote transaction body. +// See: _oasis-core/go/governance/api/api.go +type ConsensusCastVote struct { + ProposalID uint64 `json:"proposal_id"` + Vote uint8 `json:"vote"` // 0=invalid, 1=yes, 2=no, 3=abstain +} + +// ============================================================================= +// Event Types (Staking Module) +// ============================================================================= + +// ConsensusTransferEvent represents a staking transfer event. +// See: _oasis-core/go/staking/api/api.go:223-229 +type ConsensusTransferEvent struct { + From [21]byte `json:"from"` // Oasis address + To [21]byte `json:"to"` // Oasis address + Amount string `json:"amount"` // string representation of quantity.Quantity +} + +// ConsensusBurnEvent represents a staking burn event. +// See: _oasis-core/go/staking/api/api.go:236-240 +type ConsensusBurnEvent struct { + Owner [21]byte `json:"owner"` // Oasis address + Amount string `json:"amount"` // string representation of quantity.Quantity +} + +// ConsensusAddEscrowEvent represents an add escrow event. +// See: _oasis-core/go/staking/api/api.go:266-273 +type ConsensusAddEscrowEvent struct { + Owner [21]byte `json:"owner"` // Oasis address + Escrow [21]byte `json:"escrow"` // Oasis address + Amount string `json:"amount"` // string representation of quantity.Quantity + NewShares string `json:"new_shares"` // string representation of quantity.Quantity +} + +// ConsensusReclaimEscrowEvent represents a reclaim escrow event. +// See: _oasis-core/go/staking/api/api.go:313-320 +type ConsensusReclaimEscrowEvent struct { + Owner [21]byte `json:"owner"` // Oasis address + Escrow [21]byte `json:"escrow"` // Oasis address + Amount string `json:"amount"` // string representation of quantity.Quantity + Shares string `json:"shares"` // string representation of quantity.Quantity +} + +// ============================================================================= +// Tendermint Protobuf Types (minimal definitions to replace tendermint dependency) +// ============================================================================= +// Source: github.com/tendermint/tendermint v0.34.21 +// These types are used only for protobuf deserialization - no tendermint business logic is needed. + +// protoMessage is a common base type that implements proto.Message interface. +// All tendermint types embed this to satisfy proto.Unmarshal requirements. +type protoMessage struct{} + +func (protoMessage) Reset() {} +func (protoMessage) String() string { return "" } +func (protoMessage) ProtoMessage() {} + +// tmBlockStoreState represents BlockStoreState from tendermint/proto/tendermint/store/types.proto +type tmBlockStoreState struct { + protoMessage + Base int64 `protobuf:"varint,1,opt,name=base,proto3"` + Height int64 `protobuf:"varint,2,opt,name=height,proto3"` +} + +// tmBlockMeta represents BlockMeta from tendermint/proto/tendermint/types/types.proto +type tmBlockMeta struct { + protoMessage + BlockID tmBlockID `protobuf:"bytes,1,opt,name=block_id,json=blockId,proto3"` + BlockSize int64 `protobuf:"varint,2,opt,name=block_size,json=blockSize,proto3"` + Header tmHeader `protobuf:"bytes,3,opt,name=header,proto3"` + NumTxs int64 `protobuf:"varint,4,opt,name=num_txs,json=numTxs,proto3"` +} + +// tmBlockID from tendermint/proto/tendermint/types/types.proto +type tmBlockID struct { + protoMessage + Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3"` + PartSetHeader tmPartSetHeader `protobuf:"bytes,2,opt,name=part_set_header,json=partSetHeader,proto3"` +} + + +// tmPartSetHeader from tendermint/proto/tendermint/types/types.proto +type tmPartSetHeader struct { + protoMessage + Total uint32 `protobuf:"varint,1,opt,name=total,proto3"` + Hash []byte `protobuf:"bytes,2,opt,name=hash,proto3"` +} + + +// tmHeader from tendermint/proto/tendermint/types/types.proto +type tmHeader struct { + protoMessage + Version tmConsensus `protobuf:"bytes,1,opt,name=version,proto3"` + ChainID string `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3"` + Height int64 `protobuf:"varint,3,opt,name=height,proto3"` + Time time.Time `protobuf:"bytes,4,opt,name=time,proto3,stdtime"` + LastBlockID tmBlockID `protobuf:"bytes,5,opt,name=last_block_id,json=lastBlockId,proto3"` + LastCommitHash []byte `protobuf:"bytes,6,opt,name=last_commit_hash,json=lastCommitHash,proto3"` + DataHash []byte `protobuf:"bytes,7,opt,name=data_hash,json=dataHash,proto3"` + ValidatorsHash []byte `protobuf:"bytes,8,opt,name=validators_hash,json=validatorsHash,proto3"` + NextValidatorsHash []byte `protobuf:"bytes,9,opt,name=next_validators_hash,json=nextValidatorsHash,proto3"` + ConsensusHash []byte `protobuf:"bytes,10,opt,name=consensus_hash,json=consensusHash,proto3"` + AppHash []byte `protobuf:"bytes,11,opt,name=app_hash,json=appHash,proto3"` + LastResultsHash []byte `protobuf:"bytes,12,opt,name=last_results_hash,json=lastResultsHash,proto3"` + EvidenceHash []byte `protobuf:"bytes,13,opt,name=evidence_hash,json=evidenceHash,proto3"` + ProposerAddress []byte `protobuf:"bytes,14,opt,name=proposer_address,json=proposerAddress,proto3"` +} + + +// tmConsensus from tendermint/proto/tendermint/types/types.proto +type tmConsensus struct { + protoMessage + Block uint64 `protobuf:"varint,1,opt,name=block,proto3"` + App uint64 `protobuf:"varint,2,opt,name=app,proto3"` } + + +// tmPart from tendermint/proto/tendermint/types/types.proto +type tmPart struct { + protoMessage + Index uint32 `protobuf:"varint,1,opt,name=index,proto3"` + Bytes []byte `protobuf:"bytes,2,opt,name=bytes,proto3"` + Proof tmProof `protobuf:"bytes,3,opt,name=proof,proto3"` +} + + +// tmProof from tendermint/proto/tendermint/types/types.proto +type tmProof struct { + protoMessage + Total int64 `protobuf:"varint,1,opt,name=total,proto3"` + Index int64 `protobuf:"varint,2,opt,name=index,proto3"` + LeafHash []byte `protobuf:"bytes,3,opt,name=leaf_hash,json=leafHash,proto3"` + Aunts [][]byte `protobuf:"bytes,4,rep,name=aunts,proto3"` +} + + +// tmCommit from tendermint/proto/tendermint/types/types.proto +type tmCommit struct { + protoMessage + Height int64 `protobuf:"varint,1,opt,name=height,proto3"` + Round int32 `protobuf:"varint,2,opt,name=round,proto3"` + BlockID tmBlockID `protobuf:"bytes,3,opt,name=block_id,json=blockId,proto3"` + Signatures []tmCommitSig `protobuf:"bytes,4,rep,name=signatures,proto3"` +} + + +// tmCommitSig from tendermint/proto/tendermint/types/types.proto +type tmCommitSig struct { + protoMessage + BlockIDFlag int32 `protobuf:"varint,1,opt,name=block_id_flag,json=blockIdFlag,proto3"` + ValidatorAddress []byte `protobuf:"bytes,2,opt,name=validator_address,json=validatorAddress,proto3"` + Timestamp time.Time `protobuf:"bytes,3,opt,name=timestamp,proto3,stdtime"` + Signature []byte `protobuf:"bytes,4,opt,name=signature,proto3"` +} + + +// tmDuplicateVoteEvidence from tendermint/proto/tendermint/types/evidence.proto +type tmDuplicateVoteEvidence struct { + protoMessage + VoteA *tmVote `protobuf:"bytes,1,opt,name=vote_a,json=voteA,proto3"` + VoteB *tmVote `protobuf:"bytes,2,opt,name=vote_b,json=voteB,proto3"` + TotalVotingPower int64 `protobuf:"varint,3,opt,name=total_voting_power,json=totalVotingPower,proto3"` + ValidatorPower int64 `protobuf:"varint,4,opt,name=validator_power,json=validatorPower,proto3"` + Timestamp time.Time `protobuf:"bytes,5,opt,name=timestamp,proto3,stdtime"` +} + + +// tmVote from tendermint/proto/tendermint/types/types.proto +type tmVote struct { + protoMessage + Type int32 `protobuf:"varint,1,opt,name=type,proto3"` + Height int64 `protobuf:"varint,2,opt,name=height,proto3"` + Round int32 `protobuf:"varint,3,opt,name=round,proto3"` + BlockID tmBlockID `protobuf:"bytes,4,opt,name=block_id,json=blockId,proto3"` + Timestamp time.Time `protobuf:"bytes,5,opt,name=timestamp,proto3,stdtime"` + ValidatorAddress []byte `protobuf:"bytes,6,opt,name=validator_address,json=validatorAddress,proto3"` + ValidatorIndex int32 `protobuf:"varint,7,opt,name=validator_index,json=validatorIndex,proto3"` + Signature []byte `protobuf:"bytes,8,opt,name=signature,proto3"` +} + + +// tmLightClientAttackEvidence from tendermint/proto/tendermint/types/evidence.proto +type tmLightClientAttackEvidence struct { + protoMessage + ConflictingBlock *tmLightBlock `protobuf:"bytes,1,opt,name=conflicting_block,json=conflictingBlock,proto3"` + CommonHeight int64 `protobuf:"varint,2,opt,name=common_height,json=commonHeight,proto3"` + ByzantineValidators []*tmValidator `protobuf:"bytes,3,rep,name=byzantine_validators,json=byzantineValidators,proto3"` + TotalVotingPower int64 `protobuf:"varint,4,opt,name=total_voting_power,json=totalVotingPower,proto3"` + Timestamp time.Time `protobuf:"bytes,5,opt,name=timestamp,proto3,stdtime"` +} + + +// tmLightBlock from tendermint/proto/tendermint/types/types.proto +type tmLightBlock struct { + protoMessage + SignedHeader *tmSignedHeader `protobuf:"bytes,1,opt,name=signed_header,json=signedHeader,proto3"` + ValidatorSet *tmValidatorSet `protobuf:"bytes,2,opt,name=validator_set,json=validatorSet,proto3"` +} + + +// tmSignedHeader from tendermint/proto/tendermint/types/types.proto +type tmSignedHeader struct { + protoMessage + Header *tmHeader `protobuf:"bytes,1,opt,name=header,proto3"` + Commit *tmCommit `protobuf:"bytes,2,opt,name=commit,proto3"` +} + + +// tmValidatorSet from tendermint/proto/tendermint/types/validator.proto +type tmValidatorSet struct { + protoMessage + Validators []*tmValidator `protobuf:"bytes,1,rep,name=validators,proto3"` + Proposer *tmValidator `protobuf:"bytes,2,opt,name=proposer,proto3"` + TotalVotingPower int64 `protobuf:"varint,3,opt,name=total_voting_power,json=totalVotingPower,proto3"` +} + + +// tmValidator from tendermint/proto/tendermint/types/validator.proto +type tmValidator struct { + protoMessage + Address []byte `protobuf:"bytes,1,opt,name=address,proto3"` + PubKey []byte `protobuf:"bytes,2,opt,name=pub_key,json=pubKey,proto3"` + VotingPower int64 `protobuf:"varint,3,opt,name=voting_power,json=votingPower,proto3"` + ProposerPriority int64 `protobuf:"varint,4,opt,name=proposer_priority,json=proposerPriority,proto3"` +} + + +// tmABCIResponses from tendermint/proto/tendermint/state/types.proto +type tmABCIResponses struct { + protoMessage + DeliverTxs []*tmResponseDeliverTx `protobuf:"bytes,1,rep,name=deliver_txs,json=deliverTxs,proto3"` + EndBlock *tmResponseEndBlock `protobuf:"bytes,2,opt,name=end_block,json=endBlock,proto3"` + BeginBlock *tmResponseBeginBlock `protobuf:"bytes,3,opt,name=begin_block,json=beginBlock,proto3"` +} + + +// tmResponseDeliverTx from tendermint/proto/tendermint/abci/types.proto +type tmResponseDeliverTx struct { + protoMessage + Code uint32 `protobuf:"varint,1,opt,name=code,proto3"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3"` + Log string `protobuf:"bytes,3,opt,name=log,proto3"` + Info string `protobuf:"bytes,4,opt,name=info,proto3"` + GasWanted int64 `protobuf:"varint,5,opt,name=gas_wanted,json=gasWanted,proto3"` + GasUsed int64 `protobuf:"varint,6,opt,name=gas_used,json=gasUsed,proto3"` + Events []tmEvent `protobuf:"bytes,7,rep,name=events,proto3"` + Codespace string `protobuf:"bytes,8,opt,name=codespace,proto3"` +} + + +// tmResponseBeginBlock from tendermint/proto/tendermint/abci/types.proto +type tmResponseBeginBlock struct { + protoMessage + Events []tmEvent `protobuf:"bytes,1,rep,name=events,proto3"` +} + + +// tmResponseEndBlock from tendermint/proto/tendermint/abci/types.proto +type tmResponseEndBlock struct { + protoMessage + ValidatorUpdates []tmValidatorUpdate `protobuf:"bytes,1,rep,name=validator_updates,json=validatorUpdates,proto3"` + ConsensusParamUpdates *tmConsensusParams `protobuf:"bytes,2,opt,name=consensus_param_updates,json=consensusParamUpdates,proto3"` + Events []tmEvent `protobuf:"bytes,3,rep,name=events,proto3"` +} + + +// tmEvent from tendermint/proto/tendermint/abci/types.proto +type tmEvent struct { + protoMessage + Type string `protobuf:"bytes,1,opt,name=type,proto3"` + Attributes []tmEventAttribute `protobuf:"bytes,2,rep,name=attributes,proto3"` +} + + +// tmEventAttribute from tendermint/proto/tendermint/abci/types.proto +type tmEventAttribute struct { + protoMessage + Key []byte `protobuf:"bytes,1,opt,name=key,proto3"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3"` + Index bool `protobuf:"varint,3,opt,name=index,proto3"` +} + + +// tmValidatorUpdate from tendermint/proto/tendermint/abci/types.proto +type tmValidatorUpdate struct { + protoMessage + PubKey []byte `protobuf:"bytes,1,opt,name=pub_key,json=pubKey,proto3"` + Power int64 `protobuf:"varint,2,opt,name=power,proto3"` +} + + +// tmConsensusParams from tendermint/proto/tendermint/types/params.proto +type tmConsensusParams struct { + protoMessage + Block *tmBlockParams `protobuf:"bytes,1,opt,name=block,proto3"` + Evidence *tmEvidenceParams `protobuf:"bytes,2,opt,name=evidence,proto3"` + Validator *tmValidatorParams `protobuf:"bytes,3,opt,name=validator,proto3"` + Version *tmVersionParams `protobuf:"bytes,4,opt,name=version,proto3"` +} + + +// tmBlockParams from tendermint/proto/tendermint/types/params.proto +type tmBlockParams struct { + protoMessage + MaxBytes int64 `protobuf:"varint,1,opt,name=max_bytes,json=maxBytes,proto3"` + MaxGas int64 `protobuf:"varint,2,opt,name=max_gas,json=maxGas,proto3"` +} + + +// tmEvidenceParams from tendermint/proto/tendermint/types/params.proto +type tmEvidenceParams struct { + protoMessage + MaxAgeNumBlocks int64 `protobuf:"varint,1,opt,name=max_age_num_blocks,json=maxAgeNumBlocks,proto3"` + MaxAgeDuration int64 `protobuf:"varint,2,opt,name=max_age_duration,json=maxAgeDuration,proto3"` + MaxBytes int64 `protobuf:"varint,3,opt,name=max_bytes,json=maxBytes,proto3"` +} + + +// tmValidatorParams from tendermint/proto/tendermint/types/params.proto +type tmValidatorParams struct { + protoMessage + PubKeyTypes []string `protobuf:"bytes,1,rep,name=pub_key_types,json=pubKeyTypes,proto3"` +} + + +// tmVersionParams from tendermint/proto/tendermint/types/params.proto +type tmVersionParams struct { + protoMessage + AppVersion uint64 `protobuf:"varint,1,opt,name=app_version,json=appVersion,proto3"` +} + + +// tmState from tendermint/proto/tendermint/state/types.proto +type tmState struct { + protoMessage + Version uint64 `protobuf:"varint,1,opt,name=version,proto3"` + ChainID string `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3"` + InitialHeight int64 `protobuf:"varint,3,opt,name=initial_height,json=initialHeight,proto3"` + LastBlockHeight int64 `protobuf:"varint,4,opt,name=last_block_height,json=lastBlockHeight,proto3"` +} + diff --git a/badgerdb-sampler/types_evm.go b/badgerdb-sampler/types_evm.go index a660bf6..5421c52 100644 --- a/badgerdb-sampler/types_evm.go +++ b/badgerdb-sampler/types_evm.go @@ -3,34 +3,34 @@ package main // EVMDataInfo represents decoded EVM storage data. // See: _oasis-sdk/runtime-sdk/modules/evm/src/state.rs type EVMDataInfo struct { - StorageType string `json:"storage_type"` // "code", "storage", "block_hash", "confidential_storage" - Address string `json:"address,omitempty"` // H160 (20 bytes) - contract address (hex) - StorageSlot string `json:"storage_slot,omitempty"` // H256 (32 bytes) - storage slot (hex) - StorageValue string `json:"storage_value,omitempty"` // H256 (32 bytes) - storage value (hex) - Round uint64 `json:"round,omitempty"` // For block_hash type - BlockHash string `json:"block_hash,omitempty"` // H256 (32 bytes) - block hash (hex) - BytecodeSize int `json:"bytecode_size,omitempty"` // Size of contract bytecode - BytecodeRaw string `json:"bytecode_raw,omitempty"` // First 32 bytes of raw bytecode (hex) + StorageType string `json:"storage_type"` // "code", "storage", "block_hash", "confidential_storage" + Address string `json:"address,omitempty"` // H160 (20 bytes) - contract address (0x-prefixed hex) + StorageSlot string `json:"storage_slot,omitempty"` // H256 (32 bytes) - storage slot (0x-prefixed hex) + StorageValue string `json:"storage_value,omitempty"` // H256 (32 bytes) - storage value (0x-prefixed hex) + RuntimeHeight uint64 `json:"runtime_height,omitempty"` // For block_hash type (runtime block height) + BlockHash string `json:"block_hash,omitempty"` // H256 (32 bytes) - block hash (0x-prefixed hex) + BytecodeSize int `json:"bytecode_size,omitempty"` // Size of contract bytecode + BytecodeRaw string `json:"bytecode_raw,omitempty"` // First 32 bytes of raw bytecode (0x-prefixed hex) } // EVMEventInfo represents a decoded EVM Log event. // See: _oasis-sdk/runtime-sdk/modules/evm/src/lib.rs:253-263 (Event::Log) // See: _oasis-sdk/client-sdk/go/modules/evm/types.go:56-61 (Event) type EVMEventInfo struct { - Address string `json:"address"` // H160 contract address (hex, 40 chars) + Address string `json:"address"` // H160 contract address (0x-prefixed hex) TopicCount int `json:"topic_count"` // Number of topics (0-4) - Topics []string `json:"topics,omitempty"` // H256 topics array (hex, 64 chars each) + Topics []string `json:"topics,omitempty"` // H256 topics array (0x-prefixed hex) EventSignature string `json:"event_signature,omitempty"` // Human-readable signature (if known) - EventHash string `json:"event_hash,omitempty"` // topic[0] keccak256 hash + EventHash string `json:"event_hash,omitempty"` // topic[0] keccak256 hash (0x-prefixed) DataSize int `json:"data_size"` // Size of non-indexed data - DataRaw string `json:"data_raw,omitempty"` // First 64 bytes of raw data (hex) + DataRaw string `json:"data_raw,omitempty"` // First 64 bytes of raw data (0x-prefixed hex) DecodeError string `json:"decode_error,omitempty"` } // EVMTxInputInfo represents decoded EVM transaction input artifacts. // See: _oasis-core/go/runtime/transaction/transaction.go:126-140 (inputArtifacts) type EVMTxInputInfo struct { - TxHash string `json:"tx_hash"` // Transaction hash (hex, 64 chars) + TxHash string `json:"tx_hash"` // Transaction hash (0x-prefixed hex) BatchOrder uint32 `json:"batch_order"` // Order within batch InputSize int `json:"input_size"` // Raw input size Method string `json:"method,omitempty"` // SDK method (e.g., "evm.Call") @@ -41,20 +41,20 @@ type EVMTxInputInfo struct { // EVMCallInfo represents EVM-specific transaction call details. // See: _oasis-sdk/runtime-sdk/modules/evm/src/types.rs:10-16 (Call/Create) type EVMCallInfo struct { - Type string `json:"type"` // "call" or "create" - Address string `json:"address,omitempty"` // H160 target address (hex, 40 chars) - Value string `json:"value,omitempty"` // U256 value in wei (decimal string) - DataSize int `json:"data_size"` // Size of call data or init code - DataRaw string `json:"data_raw,omitempty"` // First 32 bytes of raw data (hex) + Type string `json:"type"` // "call" or "create" + Address string `json:"address,omitempty"` // H160 target address (0x-prefixed hex) + Value string `json:"value,omitempty"` // U256 value in wei (decimal string) + DataSize int `json:"data_size"` // Size of call data or init code + DataRaw string `json:"data_raw,omitempty"` // First 32 bytes of raw data (0x-prefixed hex) } // EVMTxOutputInfo represents decoded EVM transaction output artifacts. // See: _oasis-core/go/runtime/transaction/transaction.go:142-150 (outputArtifacts) type EVMTxOutputInfo struct { - OutputSize int `json:"output_size"` // Raw output size - Success bool `json:"success"` // Transaction succeeded - ResultSize int `json:"result_size,omitempty"` // Size of result data - ResultRaw string `json:"result_raw,omitempty"` // First 32 bytes of raw result (hex) - Error string `json:"error,omitempty"` // Error message if failed + OutputSize int `json:"output_size"` // Raw output size + Success bool `json:"success"` // Transaction succeeded + ResultSize int `json:"result_size,omitempty"` // Size of result data + ResultRaw string `json:"result_raw,omitempty"` // First 32 bytes of raw result (0x-prefixed hex) + Error string `json:"error,omitempty"` // Error message if failed DecodeError string `json:"decode_error,omitempty"` } diff --git a/badgerdb-sampler/types_runtime.go b/badgerdb-sampler/types_runtime.go index a6862a3..c0d79a2 100644 --- a/badgerdb-sampler/types_runtime.go +++ b/badgerdb-sampler/types_runtime.go @@ -85,11 +85,10 @@ type RuntimeOutputArtifacts struct { // RuntimeMkvsKeyInfo represents a decoded runtime-mkvs key. // See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 type RuntimeMkvsKeyInfo struct { - KeyType string `json:"key_type"` // "node", "write_log", "roots_metadata", "root_updated_nodes", "metadata", "unknown" - Height uint64 `json:"height,omitempty"` // For write_log, roots_metadata, root_updated_nodes - Hash string `json:"hash,omitempty"` // For node (hex, truncated) - DbPrefix byte `json:"db_prefix,omitempty"` // 0x01 or 0x05 if present - DecodeError string `json:"decode_error,omitempty"` + KeyType string `json:"key_type"` // "node", "write_log", "roots_metadata", "root_updated_nodes", "metadata", "unknown" + RuntimeHeight uint64 `json:"runtime_height,omitempty"` // For write_log, roots_metadata, root_updated_nodes + Hash string `json:"hash,omitempty"` // For node (hex, truncated) + DecodeError string `json:"decode_error,omitempty"` } // RuntimeMkvsNodeInfo represents a decoded runtime-mkvs value (node). @@ -124,10 +123,10 @@ type RuntimeMkvsInternalInfo struct { // RuntimeHistoryKeyInfo represents a decoded runtime-history key. // See: _oasis-core/go/runtime/history/db.go:19-31 type RuntimeHistoryKeyInfo struct { - KeyType string `json:"key_type"` // "metadata", "block", "round_results", "unknown" - Height uint64 `json:"height,omitempty"` // For block, round_results (block height/round) - ExtraData string `json:"extra,omitempty"` // Unexpected extra bytes (hex) - DecodeError string `json:"decode_error,omitempty"` + KeyType string `json:"key_type"` // "metadata", "block", "round_results", "unknown" + RuntimeHeight uint64 `json:"runtime_height,omitempty"` // For block, round_results (runtime block height) + ExtraData string `json:"extra,omitempty"` // Unexpected extra bytes (hex) + DecodeError string `json:"decode_error,omitempty"` } // RuntimeHistoryValueInfo represents a decoded runtime-history value. @@ -143,22 +142,22 @@ type RuntimeHistoryValueInfo struct { // RuntimeHistoryMetadataInfo represents decoded runtime history metadata. // See: _oasis-core/go/runtime/history/db.go:34-44 type RuntimeHistoryMetadataInfo struct { - Version uint64 `json:"version"` - RuntimeID string `json:"runtime_id"` // hex, truncated - LastRound uint64 `json:"last_round"` - LastConsensusHeight int64 `json:"last_consensus_height"` + Version uint64 `json:"version"` + RuntimeID string `json:"runtime_id"` // hex, truncated + LastRuntimeHeight uint64 `json:"last_runtime_height"` // last runtime block height + LastConsensusHeight int64 `json:"last_consensus_height"` // last consensus block height } // RuntimeHistoryBlockInfo represents a decoded runtime history block. // See: _oasis-core/go/roothash/api/api.go:402-409 (AnnotatedBlock) // See: _oasis-core/go/roothash/api/block/header.go:69-99 (Header) type RuntimeHistoryBlockInfo struct { - ConsensusHeight int64 `json:"consensus_height"` - Round uint64 `json:"round"` - Timestamp string `json:"timestamp"` // RFC3339 format + ConsensusHeight int64 `json:"consensus_height"` // consensus block height + RuntimeHeight uint64 `json:"runtime_height"` // runtime block height (formerly "round") + Timestamp string `json:"timestamp"` // RFC3339 format HeaderType string `json:"header_type"` - StateRoot string `json:"state_root,omitempty"` // hex, truncated - BlockNil bool `json:"block_nil,omitempty"` // true if block was nil + StateRoot string `json:"state_root,omitempty"` // hex, truncated + BlockNil bool `json:"block_nil,omitempty"` // true if block was nil } // RuntimeHistoryRoundResultsInfo represents decoded runtime history round results. diff --git a/badgerdb-sampler/utils.go b/badgerdb-sampler/utils.go index d3a9d2f..6285306 100644 --- a/badgerdb-sampler/utils.go +++ b/badgerdb-sampler/utils.go @@ -4,6 +4,8 @@ import ( "fmt" "math/big" "strings" + + "github.com/fxamacker/cbor/v2" ) // extractModuleName extracts module name from MKVS leaf key and returns module:subtype description @@ -193,7 +195,7 @@ func isPrintableASCII(s string) bool { return true } -// truncateHex converts bytes to hex and truncates if too long +// truncateHex converts bytes to hex and truncates if too long. func truncateHex(data []byte, maxLen int) string { hex := fmt.Sprintf("%x", data) if len(hex) > maxLen { @@ -202,6 +204,15 @@ func truncateHex(data []byte, maxLen int) string { return hex } +// truncateHex0x converts bytes to 0x-prefixed hex and truncates if too long. +func truncateHex0x(data []byte, maxLen int) string { + hex := fmt.Sprintf("%x", data) + if len(hex) > maxLen { + return "0x" + hex[:maxLen] + "..." + } + return "0x" + hex +} + // formatU256 converts big-endian U256 bytes to decimal string. func formatU256(b []byte) string { if len(b) == 0 { @@ -211,3 +222,108 @@ func formatU256(b []byte) string { return n.String() } +// bech32Charset is the character set for bech32 encoding +const bech32Charset = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" + +// bech32Encode encodes data with bech32 using the given human-readable part +func bech32Encode(hrp string, data []byte) string { + // Convert 8-bit data to 5-bit groups + var values []int + acc := 0 + bits := 0 + for _, b := range data { + acc = (acc << 8) | int(b) + bits += 8 + for bits >= 5 { + bits -= 5 + values = append(values, (acc>>bits)&31) + } + } + if bits > 0 { + values = append(values, (acc<<(5-bits))&31) + } + + // Create checksum + checksum := bech32CreateChecksum(hrp, values) + + // Build result efficiently using strings.Builder + var result strings.Builder + result.Grow(len(hrp) + 1 + len(values) + len(checksum)) + result.WriteString(hrp) + result.WriteString("1") + for _, v := range values { + result.WriteByte(bech32Charset[v]) + } + for _, v := range checksum { + result.WriteByte(bech32Charset[v]) + } + return result.String() +} + +// bech32Polymod computes the bech32 checksum polynomial +func bech32Polymod(values []int) int { + gen := []int{0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3} + chk := 1 + for _, v := range values { + b := chk >> 25 + chk = ((chk & 0x1ffffff) << 5) ^ v + for i := 0; i < 5; i++ { + if (b>>i)&1 == 1 { + chk ^= gen[i] + } + } + } + return chk +} + +// bech32HRPExpand expands the human-readable part for checksum calculation +func bech32HRPExpand(hrp string) []int { + result := make([]int, 0, len(hrp)*2+1) + for _, c := range hrp { + result = append(result, int(c)>>5) + } + result = append(result, 0) + for _, c := range hrp { + result = append(result, int(c)&31) + } + return result +} + +// bech32CreateChecksum creates the bech32 checksum +func bech32CreateChecksum(hrp string, data []int) []int { + values := append(bech32HRPExpand(hrp), data...) + values = append(values, 0, 0, 0, 0, 0, 0) + polymod := bech32Polymod(values) ^ 1 + checksum := make([]int, 6) + for i := 0; i < 6; i++ { + checksum[i] = (polymod >> (5 * (5 - i))) & 31 + } + return checksum +} + +// quantityBytesToString extracts a quantity.Quantity field from CBOR bytes and converts to string. +// This is a simplified parser that decodes the CBOR to get the nested big.Int value. +// See: _oasis-core/go/common/quantity/quantity.go:28-30 +func quantityBytesToString(cborBytes []byte, fieldName string) string { + // Decode to a generic map to extract the field + var data map[string]interface{} + if err := cbor.Unmarshal(cborBytes, &data); err != nil { + return "" + } + + // Get the field value + if val, ok := data[fieldName]; ok { + // quantity.Quantity is encoded as a byte slice containing the big.Int bytes + if qtyBytes, ok := val.([]byte); ok { + // Convert bytes to big.Int + var bi big.Int + bi.SetBytes(qtyBytes) + return bi.String() + } + // Try direct conversion + return fmt.Sprintf("%v", val) + } + + return "" +} + From 8dc750aecee63f70cec73cdaac792d807576b201 Mon Sep 17 00:00:00 2001 From: gw0 Date: Tue, 25 Nov 2025 12:04:01 +0100 Subject: [PATCH 14/22] badgerdb-sampler: Refactor consistent naming and improve error handling --- badgerdb-sampler/decode_consensus.go | 471 ++++++++++++++++----------- badgerdb-sampler/decode_evm.go | 199 +++++++---- badgerdb-sampler/decode_runtime.go | 158 +++++---- badgerdb-sampler/main.go | 72 ++-- badgerdb-sampler/types_consensus.go | 229 +++++++------ badgerdb-sampler/types_evm.go | 37 +-- badgerdb-sampler/types_runtime.go | 231 +++++++------ badgerdb-sampler/utils.go | 50 ++- 8 files changed, 844 insertions(+), 603 deletions(-) diff --git a/badgerdb-sampler/decode_consensus.go b/badgerdb-sampler/decode_consensus.go index 31b784b..539b7e1 100644 --- a/badgerdb-sampler/decode_consensus.go +++ b/badgerdb-sampler/decode_consensus.go @@ -11,9 +11,9 @@ import ( "github.com/gogo/protobuf/proto" ) -// decodeKeyConsensusBlockstore parses consensus-blockstore key and returns structured info. +// decodeConsensusBlockstoreKey parses consensus-blockstore key and returns structured info. // See: tendermint/store/store.go for key formats (H:, P:, C:, SC:, BH:) -func decodeKeyConsensusBlockstore(key []byte) ConsensusBlockstoreKeyInfo { +func decodeConsensusBlockstoreKey(key []byte) ConsensusBlockstoreKeyInfo { info := ConsensusBlockstoreKeyInfo{} if len(key) < 2 { @@ -27,9 +27,11 @@ func decodeKeyConsensusBlockstore(key []byte) ConsensusBlockstoreKeyInfo { return info } + info.KeySize = len(key) + // Parse ASCII key after 0x01 prefix asciiKey := string(key[1:]) - info.RawKey = asciiKey + info.KeyRaw = asciiKey // blockStore state key if asciiKey == "blockStore" { @@ -85,13 +87,10 @@ func decodeKeyConsensusBlockstore(key []byte) ConsensusBlockstoreKeyInfo { return info } -// decodeValueConsensusBlockstore decodes protobuf value and returns structured info. +// decodeConsensusBlockstoreValue decodes protobuf value and returns structured info. // See: tendermint/proto/tendermint/store and tendermint/proto/tendermint/types -func decodeValueConsensusBlockstore(keyType string, value []byte) ConsensusBlockstoreValueInfo { - info := ConsensusBlockstoreValueInfo{ - KeyType: keyType, - Size: len(value), - } +func decodeConsensusBlockstoreValue(keyType string, value []byte) ConsensusBlockstoreValueInfo { + info := ConsensusBlockstoreValueInfo{} switch keyType { case "blockstore_state": @@ -102,7 +101,7 @@ func decodeValueConsensusBlockstore(keyType string, value []byte) ConsensusBlock ConsensusHeight: state.Height, } } else { - info.DecodeError = err.Error() + info.StateError = err.Error() } case "block_meta": @@ -119,8 +118,8 @@ func decodeValueConsensusBlockstore(keyType string, value []byte) ConsensusBlock blockMeta.Time = meta.Header.Time.Format(time.RFC3339) } - // Format AppHash (truncate to 16 bytes = 32 hex chars) - blockMeta.AppHash = truncateHex(meta.Header.AppHash, 32) + // Format AppHash (truncate to 16 hex chars) + blockMeta.AppHash = truncateHex(meta.Header.AppHash, TruncateHashSize) // Truncate ChainID chainID := meta.Header.ChainID @@ -131,7 +130,7 @@ func decodeValueConsensusBlockstore(keyType string, value []byte) ConsensusBlock info.BlockMeta = blockMeta } else { - info.DecodeError = err.Error() + info.BlockMetaError = err.Error() } case "block_part": @@ -139,11 +138,11 @@ func decodeValueConsensusBlockstore(keyType string, value []byte) ConsensusBlock if err := proto.Unmarshal(value, &part); err == nil { info.Part = &ConsensusPartInfo{ Index: part.Index, - BytesSize: len(part.Bytes), + PartSize: len(part.Bytes), ProofTotal: part.Proof.Total, } } else { - info.DecodeError = err.Error() + info.PartError = err.Error() } case "block_commit", "seen_commit": @@ -155,58 +154,64 @@ func decodeValueConsensusBlockstore(keyType string, value []byte) ConsensusBlock Signatures: len(commit.Signatures), } } else { - info.DecodeError = err.Error() + info.CommitError = err.Error() } case "block_hash": // Block hash values are plain strings containing height numbers if height, err := strconv.ParseInt(string(value), 10, 64); err == nil { info.HashHeight = height - } else { - info.DecodeError = err.Error() } - - default: - info.DecodeError = fmt.Sprintf("unknown key type: %s", keyType) + // Note: No error field for block_hash as HashHeight is a simple int64 field } return info } -// decodeKeyConsensusEvidence parses consensus-evidence key and returns structured info. -func decodeKeyConsensusEvidence(key []byte) ConsensusEvidenceKeyInfo { - info := ConsensusEvidenceKeyInfo{} +// decodeConsensusEvidenceKey parses consensus-evidence key and returns structured info. +func decodeConsensusEvidenceKey(key []byte) ConsensusEvidenceKeyInfo { + info := ConsensusEvidenceKeyInfo{ + KeySize: len(key), + KeyHex: truncateHex(key, TruncateLongSize), + } if len(key) < 2 { info.KeyType = "unknown" - info.RawKey = fmt.Sprintf("%x", key) return info } // All keys start with 0x01 (dbVersion from Oasis BadgerDB wrapper) if key[0] != 0x01 { info.KeyType = "unknown" - info.RawKey = fmt.Sprintf("%x", key) return info } // Evidence DB is typically empty or uses simple key patterns info.KeyType = fmt.Sprintf("type_%02x", key[1]) info.PrefixByte = key[1] - info.RawKey = fmt.Sprintf("%x", key[1:]) return info } -// decodeValueConsensusEvidence decodes evidence value and returns structured info. -func decodeValueConsensusEvidence(keyType string, value []byte) ConsensusEvidenceValueInfo { - info := ConsensusEvidenceValueInfo{ - Size: len(value), +// decodeConsensusEvidenceValue decodes evidence value and returns structured info. +// Returns (info, error) where error is a parsing error. +func decodeConsensusEvidenceValue(keyType string, value []byte) (*ConsensusEvidenceValueInfo, string) { + info := &ConsensusEvidenceValueInfo{ + ValueSize: len(value), + ValueHex: truncateHex(value, TruncateLongSize), } if len(value) == 0 { - return info + return info, "" } + // Recover from protobuf panics that occur with schema mismatches + var panicErr string + defer func() { + if r := recover(); r != nil { + panicErr = fmt.Sprintf("protobuf panic: %v", r) + } + }() + // Try to decode as DuplicateVoteEvidence first var dve tmDuplicateVoteEvidence if err := proto.Unmarshal(value, &dve); err == nil && dve.VoteA != nil { @@ -220,7 +225,7 @@ func decodeValueConsensusEvidence(keyType string, value []byte) ConsensusEvidenc if dve.Timestamp.Unix() > 0 { info.Timestamp = dve.Timestamp.Format(time.RFC3339) } - return info + return info, panicErr } // Try to decode as LightClientAttackEvidence @@ -231,26 +236,28 @@ func decodeValueConsensusEvidence(keyType string, value []byte) ConsensusEvidenc if lca.Timestamp.Unix() > 0 { info.Timestamp = lca.Timestamp.Format(time.RFC3339) } - return info + return info, panicErr } // If all parsing failed, show raw value info.EvidenceType = "unknown" - info.RawValue = truncateHex(value, 64) - info.DecodeError = "unable to decode evidence format" - - return info + if panicErr != "" { + return info, panicErr + } + return info, "unable to decode evidence format" } -// decodeKeyConsensusMkvs parses consensus-mkvs key and returns structured info. +// decodeConsensusMkvsKey parses consensus-mkvs key and returns structured info. // Keys use keyformat encoding: [type_byte][data...] // See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 -func decodeKeyConsensusMkvs(key []byte) ConsensusMkvsKeyInfo { - info := ConsensusMkvsKeyInfo{} +func decodeConsensusMkvsKey(key []byte) ConsensusMkvsKeyInfo { + info := ConsensusMkvsKeyInfo{ + KeySize: len(key), + KeyHex: truncateHex(key, TruncateLongSize), + } if len(key) < 1 { info.KeyType = "unknown" - info.DecodeError = "key too short" return info } @@ -261,7 +268,7 @@ func decodeKeyConsensusMkvs(key []byte) ConsensusMkvsKeyInfo { switch prefixByte { case 0x00: info.KeyType = "node" - info.Hash = truncateHex(data, 16) + info.Hash = truncateHex(data, TruncateHashSize) case 0x01: info.KeyType = "write_log" @@ -269,18 +276,14 @@ func decodeKeyConsensusMkvs(key []byte) ConsensusMkvsKeyInfo { info.ConsensusHeight = binary.BigEndian.Uint64(data[0:8]) if len(data) >= 8+33 { info.RootType = data[8] - info.Hash = truncateHex(data[9:9+32], 32) + info.Hash = truncateHex(data[9:9+32], TruncateHashSize) } - } else { - info.DecodeError = "write_log data too short" } case 0x02: info.KeyType = "roots_metadata" if len(data) >= 8 { info.ConsensusHeight = binary.BigEndian.Uint64(data[0:8]) - } else { - info.DecodeError = "roots_metadata data too short" } case 0x03: @@ -289,10 +292,8 @@ func decodeKeyConsensusMkvs(key []byte) ConsensusMkvsKeyInfo { info.ConsensusHeight = binary.BigEndian.Uint64(data[0:8]) if len(data) >= 8+33 { info.RootType = data[8] - info.Hash = truncateHex(data[9:9+32], 32) + info.Hash = truncateHex(data[9:9+32], TruncateHashSize) } - } else { - info.DecodeError = "root_updated_nodes data too short" } case 0x04: @@ -302,32 +303,33 @@ func decodeKeyConsensusMkvs(key []byte) ConsensusMkvsKeyInfo { info.KeyType = "multipart_restore_log" if len(data) >= 33 { info.RootType = data[0] - info.Hash = truncateHex(data[1:33], 16) + info.Hash = truncateHex(data[1:33], TruncateHashSize) } else { - info.Hash = truncateHex(data, 16) + info.Hash = truncateHex(data, TruncateHashSize) } case 0x06: info.KeyType = "root_node" if len(data) >= 33 { info.RootType = data[0] - info.Hash = truncateHex(data[1:33], 16) + info.Hash = truncateHex(data[1:33], TruncateHashSize) } else { - info.Hash = truncateHex(data, 16) + info.Hash = truncateHex(data, TruncateHashSize) } default: info.KeyType = fmt.Sprintf("unknown_%02x", prefixByte) - info.DecodeError = fmt.Sprintf("unknown key prefix 0x%02x", prefixByte) } return info } -// decodeValueConsensusMkvs decodes consensus MKVS value and returns structured info. +// decodeConsensusMkvsValue decodes consensus MKVS value and returns structured info. // See: _oasis-core/go/storage/mkvs/node/node.go:26-32 (prefixes), 294-309 (InternalNode), 531-537 (LeafNode) -func decodeValueConsensusMkvs(keyType string, value []byte) ConsensusMkvsNodeInfo { - info := ConsensusMkvsNodeInfo{Size: len(value)} +func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueInfo { + info := ConsensusMkvsValueInfo{ + NodeSize: len(value), + } if len(value) == 0 { info.NodeType = "empty" @@ -339,6 +341,9 @@ func decodeValueConsensusMkvs(keyType string, value []byte) ConsensusMkvsNodeInf return info } + // Add raw node hex dump + info.NodeHex = truncateHex(value, TruncateLongSize) + // Parse MKVS node: 0x00=leaf, 0x01=internal, 0x02=nil switch value[0] { case 0x00: // LeafNode @@ -346,40 +351,40 @@ func decodeValueConsensusMkvs(keyType string, value []byte) ConsensusMkvsNodeInf data := value[1:] if len(data) < 2 { - info.DecodeError = "key length missing" + info.LeafError = "key length missing" return info } - keyLen := int(binary.LittleEndian.Uint16(data[0:2])) + keySize := int(binary.LittleEndian.Uint16(data[0:2])) data = data[2:] - if len(data) < keyLen { - info.DecodeError = fmt.Sprintf("key truncated (expected %d bytes)", keyLen) + if len(data) < keySize { + info.LeafError = fmt.Sprintf("key truncated (expected %d bytes)", keySize) return info } - key := data[:keyLen] - data = data[keyLen:] + key := data[:keySize] + data = data[keySize:] // Decode consensus module key prefix module := decodeConsensusModulePrefix(key) leaf := &ConsensusMkvsLeafInfo{ - Module: module, - KeyLen: keyLen, - Key: truncateHex(key, 32), + Module: module, + KeySize: keySize, + KeyHex: truncateHex(key, TruncateLongSize), } // Extract Oasis address for staking-related modules // Staking keys have format: module_prefix (1 byte) + 21-byte Oasis address - if keyLen == 22 { + if keySize == 22 { switch key[0] { case 0x34, 0x35, 0x36, 0x37: // staking accounts, delegations, debonding, allowances leaf.OasisAddress = bech32Encode("oasis", key[1:22]) } } // Entity/node keys also contain addresses - if keyLen >= 22 { + if keySize >= 22 { switch key[0] { case 0x40, 0x41: // registry entities, nodes leaf.OasisAddress = bech32Encode("oasis", key[1:22]) @@ -387,29 +392,28 @@ func decodeValueConsensusMkvs(keyType string, value []byte) ConsensusMkvsNodeInf } if len(data) < 4 { - info.DecodeError = "value length missing" + info.LeafError = "value length missing" info.Leaf = leaf return info } - valueLen := int(binary.LittleEndian.Uint32(data[0:4])) + valueSize := int(binary.LittleEndian.Uint32(data[0:4])) data = data[4:] - leaf.ValueLen = valueLen + leaf.ValueSize = valueSize - if len(data) < valueLen { - info.DecodeError = fmt.Sprintf("value truncated (expected %d bytes)", valueLen) + if len(data) < valueSize { + info.LeafError = fmt.Sprintf("value truncated (expected %d bytes)", valueSize) info.Leaf = leaf return info } - leafValue := data[:valueLen] + leafValue := data[:valueSize] + leaf.ValueHex = truncateHex(leafValue, TruncateLongSize) // Try CBOR decode var decoded interface{} if err := cbor.Unmarshal(leafValue, &decoded); err == nil { - leaf.DecodedValue = formatCBOR(decoded, valueLen) - } else { - leaf.DecodedValue = fmt.Sprintf("binary(%d bytes)", valueLen) + leaf.ValueFormatted = formatCBOR(decoded, valueSize) } info.Leaf = leaf @@ -420,7 +424,7 @@ func decodeValueConsensusMkvs(keyType string, value []byte) ConsensusMkvsNodeInf data := value[1:] if len(data) < 2 { - info.DecodeError = "label bits missing" + info.InternalError = "label bits missing" return info } @@ -429,7 +433,7 @@ func decodeValueConsensusMkvs(keyType string, value []byte) ConsensusMkvsNodeInf labelBytes := (int(labelBits) + 7) / 8 if len(data) < labelBytes+1 { - info.DecodeError = "label truncated" + info.InternalError = "label truncated" info.Internal = &ConsensusMkvsInternalInfo{LabelBits: labelBits} return info } @@ -437,18 +441,55 @@ func decodeValueConsensusMkvs(keyType string, value []byte) ConsensusMkvsNodeInf data = data[labelBytes:] // skip label internal := &ConsensusMkvsInternalInfo{LabelBits: labelBits} - hasLeaf := data[0] == 0x00 - internal.HasLeaf = hasLeaf - if !hasLeaf { + // Check for embedded leaf node or nil marker + if len(data) < 1 { + info.InternalError = "missing leaf/nil marker" + info.Internal = internal + return info + } + + if data[0] == 0x02 { // NilNode marker - no embedded leaf + internal.HasLeaf = false data = data[1:] // skip nil marker - if len(data) >= 32 { - internal.LeftHash = truncateHex(data[:32], 16) - data = data[32:] - } - if len(data) >= 32 { - internal.RightHash = truncateHex(data[:32], 16) - } + } else if data[0] == 0x00 { // LeafNode prefix - embedded leaf present + internal.HasLeaf = true + // Skip embedded leaf: prefix(1) + keyLen(2) + key + valueLen(4) + value + data = data[1:] // skip prefix + if len(data) < 2 { + info.InternalError = "embedded leaf key length missing" + info.Internal = internal + return info + } + keyLen := binary.LittleEndian.Uint16(data[0:2]) + data = data[2:] + if len(data) < int(keyLen)+4 { + info.InternalError = "embedded leaf truncated" + info.Internal = internal + return info + } + data = data[keyLen:] // skip key + valueLen := binary.LittleEndian.Uint32(data[0:4]) + data = data[4:] + if len(data) < int(valueLen) { + info.InternalError = "embedded leaf value truncated" + info.Internal = internal + return info + } + data = data[valueLen:] // skip value + } else { + info.InternalError = fmt.Sprintf("unexpected marker 0x%02x (expected 0x00 or 0x02)", data[0]) + info.Internal = internal + return info + } + + // Read left and right hashes + if len(data) >= 32 { + internal.LeftHash = truncateHex(data[:32], TruncateHashSize) + data = data[32:] + } + if len(data) >= 32 { + internal.RightHash = truncateHex(data[:32], TruncateHashSize) } info.Internal = internal @@ -460,7 +501,7 @@ func decodeValueConsensusMkvs(keyType string, value []byte) ConsensusMkvsNodeInf default: info.NodeType = "unknown" - info.DecodeError = fmt.Sprintf("unknown prefix 0x%02x", value[0]) + info.NodeError = fmt.Sprintf("unknown prefix 0x%02x", value[0]) return info } } @@ -569,8 +610,8 @@ func decodeConsensusModulePrefix(key []byte) string { } } -// decodeKeyConsensusState parses consensus-state key and returns structured info. -func decodeKeyConsensusState(key []byte) ConsensusStateKeyInfo { +// decodeConsensusStateKey parses consensus-state key and returns structured info. +func decodeConsensusStateKey(key []byte) ConsensusStateKeyInfo { info := ConsensusStateKeyInfo{} if len(key) < 2 { @@ -584,8 +625,11 @@ func decodeKeyConsensusState(key []byte) ConsensusStateKeyInfo { return info } + info.KeySize = len(key) + // Parse ASCII key after 0x01 prefix decoded := string(key[1:]) + info.KeyHex = truncateHex(key, TruncateLongSize) // Check if it's printable ASCII if !isPrintableASCII(decoded) { @@ -594,8 +638,6 @@ func decodeKeyConsensusState(key []byte) ConsensusStateKeyInfo { return info } - info.DecodedKey = decoded - // Extract key type prefix before colon colonIdx := strings.IndexByte(decoded, ':') if colonIdx != -1 { @@ -639,234 +681,273 @@ func decodeKeyConsensusState(key []byte) ConsensusStateKeyInfo { return info } -// decodeValueConsensusState decodes state value and returns structured info. -func decodeValueConsensusState(keyType string, value []byte) ConsensusStateValueInfo { +// decodeConsensusStateValue decodes state value and returns structured info. +func decodeConsensusStateValue(keyType string, value []byte) ConsensusStateValueInfo { info := ConsensusStateValueInfo{ - KeyType: keyType, - Size: len(value), + ValueSize: len(value), + ValueHex: truncateHex(value, TruncateLongSize), } if len(value) == 0 { return info } + // Add panic recovery for all protobuf unmarshaling operations + defer func() { + if r := recover(); r != nil { + // Protobuf panic occurred (likely schema mismatch) + errMsg := fmt.Sprintf("protobuf panic: %v", r) + // Set appropriate error field based on key type + switch keyType { + case "abci_responses": + if info.ABCIResponseError == "" { + info.ABCIResponseError = errMsg + } + case "consensus_params": + if info.ConsensusParamsError == "" { + info.ConsensusParamsError = errMsg + } + case "validators": + if info.ValidatorsError == "" { + info.ValidatorsError = errMsg + } + case "state", "genesis": + if info.StateError == "" { + info.StateError = errMsg + } + } + } + }() + switch keyType { case "abci_responses": // Try to decode ABCI responses (ResponseFinalizeBlock in newer Tendermint) var resp tmABCIResponses if err := proto.Unmarshal(value, &resp); err == nil { - abciInfo := &ABCIResponseInfo{} - - // Count deliver_tx results (transaction results) - if resp.DeliverTxs != nil { - abciInfo.TxResultCount = len(resp.DeliverTxs) - } + // Validate that unmarshal actually decoded meaningful data + if resp.DeliverTxs == nil && resp.BeginBlock == nil && resp.EndBlock == nil { + info.ABCIResponseError = "protobuf unmarshal succeeded but all fields are nil (schema mismatch)" + } else { + abciResponseInfo := &ConsensusABCIResponseInfo{} + + // Count deliver_tx results (transaction results) + if resp.DeliverTxs != nil { + abciResponseInfo.TxResultCount = len(resp.DeliverTxs) + } - // Count events from end_block - if resp.EndBlock != nil { - abciInfo.EventCount = len(resp.EndBlock.Events) - if resp.EndBlock.ValidatorUpdates != nil { - abciInfo.ValidatorUpdates = len(resp.EndBlock.ValidatorUpdates) + // Count events from end_block + if resp.EndBlock != nil { + abciResponseInfo.EventCount = len(resp.EndBlock.Events) + if resp.EndBlock.ValidatorUpdates != nil { + abciResponseInfo.ValidatorUpdates = len(resp.EndBlock.ValidatorUpdates) + } } - } - // Count event types from all sources - eventSummary := &ConsensusEventSummary{ - EventTypeCounts: make(map[string]int), - } + // Count event types from all sources + eventSummary := &ConsensusEventSummary{ + EventTypeCounts: make(map[string]int), + } - // Decode transactions and collect events from DeliverTxs - txSummary := &ConsensusTransactionSummary{ - MethodCounts: make(map[string]int), - } + // Decode transactions and collect events from DeliverTxs + txSummary := &ConsensusTransactionSummary{ + MethodCounts: make(map[string]int), + } - if resp.DeliverTxs != nil { - for _, txResult := range resp.DeliverTxs { - // Count events - for _, event := range txResult.Events { - eventSummary.EventTypeCounts[event.Type]++ - } + if resp.DeliverTxs != nil { + for _, txResult := range resp.DeliverTxs { + // Count events + for _, event := range txResult.Events { + eventSummary.EventTypeCounts[event.Type]++ + } - // Decode transaction if present - if len(txResult.Data) > 0 { - decodedTx := decodeCBORTransaction(txResult.Data) - if decodedTx.DecodeError == "" { - txSummary.MethodCounts[decodedTx.Method]++ - // Only include first few transactions for sample - if len(txSummary.Transactions) < 5 { - txSummary.Transactions = append(txSummary.Transactions, decodedTx) + // Decode transaction if present + if len(txResult.Data) > 0 { + decodedTx, err := decodeCBORTransaction(txResult.Data) + if err == "" { + txSummary.MethodCounts[decodedTx.Method]++ + // Only include first few transactions for sample + if len(txSummary.Transactions) < 5 { + txSummary.Transactions = append(txSummary.Transactions, decodedTx) + } } } } } - } - if len(txSummary.MethodCounts) > 0 { - abciInfo.TransactionSummary = txSummary - } + if len(txSummary.MethodCounts) > 0 { + abciResponseInfo.TransactionSummary = txSummary + } - // Events from BeginBlock - if resp.BeginBlock != nil { - for _, event := range resp.BeginBlock.Events { - eventSummary.EventTypeCounts[event.Type]++ + // Events from BeginBlock + if resp.BeginBlock != nil { + for _, event := range resp.BeginBlock.Events { + eventSummary.EventTypeCounts[event.Type]++ + } } - } - // Events from EndBlock - if resp.EndBlock != nil { - for _, event := range resp.EndBlock.Events { - eventSummary.EventTypeCounts[event.Type]++ + // Events from EndBlock + if resp.EndBlock != nil { + for _, event := range resp.EndBlock.Events { + eventSummary.EventTypeCounts[event.Type]++ + } } - } - if len(eventSummary.EventTypeCounts) > 0 { - abciInfo.EventSummary = eventSummary - } + if len(eventSummary.EventTypeCounts) > 0 { + abciResponseInfo.EventSummary = eventSummary + } - info.ABCIInfo = abciInfo + info.ABCIResponse = abciResponseInfo + } } else { - info.DecodeError = fmt.Sprintf("failed to decode ABCI responses: %v", err) + info.ABCIResponseError = fmt.Sprintf("failed to decode ABCI responses: %v", err) } case "consensus_params": - info.RawValue = truncateHex(value, 32) var params tmConsensusParams if err := proto.Unmarshal(value, ¶ms); err == nil { - info.ConsensusParams = ¶ms + // Validate that unmarshal decoded meaningful data + if params.Block == nil && params.Evidence == nil && params.Validator == nil && params.Version == nil { + info.ConsensusParamsError = "protobuf unmarshal succeeded but all fields are nil (schema mismatch)" + } else { + info.ConsensusParams = ¶ms + } } else { - info.DecodeError = fmt.Sprintf("failed to decode consensus params: %v", err) + info.ConsensusParamsError = fmt.Sprintf("failed to decode consensus params: %v", err) } case "validators": - info.RawValue = truncateHex(value, 32) var valSet tmValidatorSet if err := proto.Unmarshal(value, &valSet); err == nil { - info.ConsensusValidators = &valSet + // Validate that unmarshal decoded meaningful data + if len(valSet.Validators) == 0 && valSet.Proposer == nil && valSet.TotalVotingPower == 0 { + info.ValidatorsError = "protobuf unmarshal succeeded but all fields are empty/nil (schema mismatch)" + } else { + info.ConsensusValidators = &valSet + } } else { - info.DecodeError = fmt.Sprintf("failed to decode validator set: %v", err) + info.ValidatorsError = fmt.Sprintf("failed to decode validator set: %v", err) } case "state": - info.RawValue = truncateHex(value, 32) var state tmState if err := proto.Unmarshal(value, &state); err == nil { - info.ConsensusState = &state + // Validate that unmarshal decoded meaningful data + if state.ChainID == "" && state.LastBlockHeight == 0 && state.InitialHeight == 0 { + info.StateError = "protobuf unmarshal succeeded but all fields are empty/zero (schema mismatch)" + } else { + info.ConsensusState = &state + } } else { - info.DecodeError = fmt.Sprintf("failed to decode state: %v", err) + info.StateError = fmt.Sprintf("failed to decode state: %v", err) } case "genesis": - info.RawValue = truncateHex(value, 64) if value[0] != '{' { - info.DecodeError = "genesis document not in expected JSON format" + info.StateError = "genesis document not in expected JSON format" } - // Genesis is JSON, RawValue shows hex preview - - default: - info.RawValue = truncateHex(value, 32) + // Genesis is JSON, ValueRaw shows hex preview (already set above) } return info } -// decodeCBORTransaction decodes a raw CBOR-encoded transaction bytes into ConsensusDecodedTransaction. +// decodeCBORTransaction decodes a raw CBOR-encoded transaction bytes into ConsensusTransactionInfo. +// Returns (decoded, error) where error is a parsing error. // See: _oasis-core/go/consensus/api/transaction/transaction.go:42-54 // See: _oasis-core/go/common/crypto/signature/signature.go:415-421 // See: _oasis-core/go/staking/api/api.go for transaction body types -func decodeCBORTransaction(rawTx []byte) ConsensusDecodedTransaction { - decoded := ConsensusDecodedTransaction{} +func decodeCBORTransaction(rawTx []byte) (ConsensusTransactionInfo, string) { + decoded := ConsensusTransactionInfo{} // Decode SignedTransaction envelope - var signedTx ConsensusSignedTransaction + var signedTx cborConsensusSignedTransaction if err := cbor.Unmarshal(rawTx, &signedTx); err != nil { - decoded.DecodeError = fmt.Sprintf("failed to decode SignedTransaction: %v", err) - return decoded + return decoded, fmt.Sprintf("failed to decode SignedTransaction: %v", err) } // Extract signer public key - decoded.Signer = fmt.Sprintf("%x", signedTx.Signature.PublicKey[:]) + decoded.Signer = truncateHex(signedTx.Signature.PublicKey[:], TruncateLongSize) // Decode inner Transaction from the blob - var tx struct { - Nonce uint64 `cbor:"nonce"` - Fee *ConsensusFee `cbor:"fee"` - Method string `cbor:"method"` - Body cbor.RawMessage `cbor:"body"` - } + var tx cborConsensusInnerTransaction if err := cbor.Unmarshal(signedTx.Blob, &tx); err != nil { - decoded.DecodeError = fmt.Sprintf("failed to decode Transaction: %v", err) - return decoded + return decoded, fmt.Sprintf("failed to decode Transaction: %v", err) } decoded.Nonce = tx.Nonce decoded.Method = tx.Method - decoded.Fee = tx.Fee + if tx.Fee != nil { + decoded.Fee = &ConsensusFee{ + Amount: tx.Fee.Amount, + Gas: tx.Fee.Gas, + } + } // Decode body based on method if len(tx.Body) > 0 { - decoded.BodyPreview = truncateHex(tx.Body, 32) + decoded.BodyPreview = truncateHex(tx.Body, TruncateLongSize) switch tx.Method { case "staking.Transfer": - var transfer ConsensusTransfer + var transfer cborConsensusTransfer if err := cbor.Unmarshal(tx.Body, &transfer); err == nil { transfer.Amount = quantityBytesToString(tx.Body, "amount") decoded.DecodedBody = transfer } case "staking.Burn": - var burn ConsensusBurn + var burn cborConsensusBurn if err := cbor.Unmarshal(tx.Body, &burn); err == nil { burn.Amount = quantityBytesToString(tx.Body, "amount") decoded.DecodedBody = burn } case "staking.AddEscrow": - var escrow ConsensusAddEscrow + var escrow cborConsensusAddEscrow if err := cbor.Unmarshal(tx.Body, &escrow); err == nil { escrow.Amount = quantityBytesToString(tx.Body, "amount") decoded.DecodedBody = escrow } case "staking.ReclaimEscrow": - var reclaim ConsensusReclaimEscrow + var reclaim cborConsensusReclaimEscrow if err := cbor.Unmarshal(tx.Body, &reclaim); err == nil { reclaim.Shares = quantityBytesToString(tx.Body, "shares") decoded.DecodedBody = reclaim } case "registry.RegisterEntity": - var regEntity ConsensusRegisterEntity + var regEntity cborConsensusRegisterEntity if err := cbor.Unmarshal(tx.Body, ®Entity); err == nil { decoded.DecodedBody = regEntity } case "registry.RegisterNode": - var regNode ConsensusRegisterNode + var regNode cborConsensusRegisterNode if err := cbor.Unmarshal(tx.Body, ®Node); err == nil { decoded.DecodedBody = regNode } case "roothash.ExecutorCommit": - var execCommit ConsensusExecutorCommit + var execCommit cborConsensusExecutorCommit if err := cbor.Unmarshal(tx.Body, &execCommit); err == nil { decoded.DecodedBody = execCommit } case "governance.SubmitProposal": - var proposal ConsensusSubmitProposal + var proposal cborConsensusSubmitProposal if err := cbor.Unmarshal(tx.Body, &proposal); err == nil { proposal.Deposit = quantityBytesToString(tx.Body, "deposit") decoded.DecodedBody = proposal } case "governance.CastVote": - var vote ConsensusCastVote + var vote cborConsensusCastVote if err := cbor.Unmarshal(tx.Body, &vote); err == nil { decoded.DecodedBody = vote } } } - return decoded + return decoded, "" } diff --git a/badgerdb-sampler/decode_evm.go b/badgerdb-sampler/decode_evm.go index e01afeb..edc6156 100644 --- a/badgerdb-sampler/decode_evm.go +++ b/badgerdb-sampler/decode_evm.go @@ -41,31 +41,27 @@ func decodeEVMData(module string, key []byte, value []byte) *EVMDataInfo { case 0x01: // CODES: evm + 0x01 + H160 (address) evmInfo.StorageType = "code" if len(data) >= 20 { - evmInfo.Address = fmt.Sprintf("0x%x", data[:20]) + evmInfo.Address = truncateHex0x(data[:20], TruncateLongSize) data = data[20:] } // Value is contract bytecode - evmInfo.BytecodeSize = len(value) + evmInfo.CodeSize = len(value) if len(value) > 0 { - previewLen := 32 - if len(value) < previewLen { - previewLen = len(value) - } - evmInfo.BytecodeRaw = fmt.Sprintf("0x%x", value[:previewLen]) + evmInfo.CodeHex = truncateHex(value, TruncateLongSize) } case 0x02: // STORAGES: evm + 0x02 + H160 (address) + H256 (slot) evmInfo.StorageType = "storage" if len(data) >= 20 { - evmInfo.Address = fmt.Sprintf("0x%x", data[:20]) + evmInfo.Address = truncateHex0x(data[:20], TruncateLongSize) data = data[20:] if len(data) >= 32 { - evmInfo.StorageSlot = fmt.Sprintf("0x%x", data[:32]) + evmInfo.StorageSlot = truncateHex0x(data[:32], TruncateHashSize) } } // Value is H256 storage value if len(value) == 32 { - evmInfo.StorageValue = fmt.Sprintf("0x%x", value) + evmInfo.StorageValue = truncateHex0x(value, TruncateLongSize) } case 0x03: // BLOCK_HASHES: evm + 0x03 + RuntimeHeight (uint64 BE) @@ -75,16 +71,16 @@ func decodeEVMData(module string, key []byte, value []byte) *EVMDataInfo { } // Value is H256 block hash if len(value) == 32 { - evmInfo.BlockHash = fmt.Sprintf("0x%x", value) + evmInfo.BlockHash = truncateHex0x(value, TruncateHashSize) } case 0x04: // CONFIDENTIAL_STORAGES: evm + 0x04 + H160 (address) + H256 (slot) evmInfo.StorageType = "confidential_storage" if len(data) >= 20 { - evmInfo.Address = fmt.Sprintf("0x%x", data[:20]) + evmInfo.Address = truncateHex0x(data[:20], TruncateLongSize) data = data[20:] if len(data) >= 32 { - evmInfo.StorageSlot = fmt.Sprintf("0x%x", data[:32]) + evmInfo.StorageSlot = truncateHex0x(data[:32], TruncateHashSize) } } // Value is encrypted - we can only show size @@ -101,32 +97,30 @@ func decodeEVMData(module string, key []byte, value []byte) *EVMDataInfo { // decodeEVMEvent decodes an EVM Log event from CBOR-encoded value. // See: _oasis-sdk/runtime-sdk/modules/evm/src/lib.rs:253-263 -func decodeEVMEvent(value []byte) *EVMEventInfo { +// Returns (info, error) where error is a parsing error, not an execution error. +func decodeEVMEvent(value []byte) (*EVMEventInfo, string) { info := &EVMEventInfo{} // Value is CBOR: [event_code, {address, topics, data}] var eventWrapper []interface{} if err := cbor.Unmarshal(value, &eventWrapper); err != nil { - info.DecodeError = err.Error() - return info + return nil, err.Error() } if len(eventWrapper) < 2 { - info.DecodeError = "invalid format" - return info + return nil, "invalid format" } eventData, ok := eventWrapper[1].(map[interface{}]interface{}) if !ok { - info.DecodeError = "invalid format" - return info + return nil, "invalid format" } // Extract address (H160) if addrBytes, ok := eventData["address"].([]byte); ok && len(addrBytes) == 20 { - info.Address = fmt.Sprintf("0x%x", addrBytes) + info.Address = truncateHex0x(addrBytes, TruncateLongSize) } else { - // Continue decoding even with invalid address - topics and data may still be useful - info.DecodeError = "invalid address (continuing decode)" + // Invalid address, but continue decoding - return partial data with error + return info, "invalid address (partial decode)" } // Extract topics (Vec) @@ -134,11 +128,11 @@ func decodeEVMEvent(value []byte) *EVMEventInfo { info.TopicCount = len(topicsArray) for i, topic := range topicsArray { if topicBytes, ok := topic.([]byte); ok && len(topicBytes) == 32 { - topicHex := fmt.Sprintf("%x", topicBytes) - info.Topics = append(info.Topics, "0x"+topicHex) + topicHex := truncateHex0x(topicBytes, TruncateHashSize) + info.Topics = append(info.Topics, topicHex) if i == 0 { - info.EventHash = "0x" + topicHex - if sig, found := EVMEventSignatures[topicHex]; found { + info.EventHash = topicHex + if sig, found := EVMEventSignatures[topicHex[:len(topicHex)]]; found { info.EventSignature = sig } } @@ -149,27 +143,109 @@ func decodeEVMEvent(value []byte) *EVMEventInfo { // Extract data if dataBytes, ok := eventData["data"].([]byte); ok { info.DataSize = len(dataBytes) - info.DataRaw = truncateHex0x(dataBytes, 128) // 64 bytes = 128 hex chars + info.DataHex = truncateHex(dataBytes, TruncateLongSize) } - return info + return info, "" } // decodeEVMTxInput decodes EVM transaction input artifacts. -func decodeEVMTxInput(key []byte, value []byte) *EVMTxInputInfo { - info := &EVMTxInputInfo{InputSize: len(value)} +// Returns (info, error) where error is a parsing error. +func decodeEVMTxInput(key []byte, value []byte) (*EVMTxInputInfo, string) { + info := &EVMTxInputInfo{ + TxInputSize: len(value), + TxInputHex: truncateHex(value, TruncateLongSize), + } if len(key) >= 33 { - info.TxHash = fmt.Sprintf("0x%x", key[1:33]) + info.TxHash = truncateHex0x(key[1:33], TruncateHashSize) } - var ia RuntimeInputArtifacts + var ia cborRuntimeInputArtifacts if err := cbor.Unmarshal(value, &ia); err != nil { - info.DecodeError = err.Error() - return info + return nil, fmt.Sprintf("failed to unmarshal RuntimeInputArtifacts: %v", err) } info.BatchOrder = ia.BatchOrder + // Try decoding as array format first (old runtime version) + var callArray cborRuntimeCallArrayFormat + if err := cbor.Unmarshal(ia.Input, &callArray); err == nil { + info.Method = callArray.Method + // Decode EVM call methods that have body structure + if callArray.Method == "evm.Call" || callArray.Method == "evm.Create" || callArray.Method == "evm.SimulateCall" || callArray.Method == "evm.EstimateGas" { + isCreate := callArray.Method == "evm.Create" + info.EVMCall = decodeEVMCallBodyFromCBOR(callArray.Body, isCreate) + } + return info, "" + } + + // Try decoding as map format (newer runtime version) + var callMap cborRuntimeCallMapFormat + if err := cbor.Unmarshal(ia.Input, &callMap); err == nil { + info.Method = callMap.Method + // Decode EVM call methods that have body structure + if callMap.Method == "evm.Call" || callMap.Method == "evm.Create" || callMap.Method == "evm.SimulateCall" || callMap.Method == "evm.EstimateGas" { + isCreate := callMap.Method == "evm.Create" + info.EVMCall = decodeEVMCallBodyFromCBOR(callMap.Body, isCreate) + } + return info, "" + } + + // Fallback: try generic array decode (for compatibility with other runtime versions) + var callArrayGeneric []interface{} + if err := cbor.Unmarshal(ia.Input, &callArrayGeneric); err == nil { + // Array format: [format, method, body, ...] or [format, [method, body, ...]] + if len(callArrayGeneric) < 2 { + return info, fmt.Sprintf("array format: insufficient fields (len=%d), expected at least 2", len(callArrayGeneric)) + } + + // Extract method and body - handle both flat and nested array formats + var method string + var bodyBytes []byte + + if methodVal, ok := callArrayGeneric[1].(string); ok { + // Flat format: [format, method, body, ...] + method = methodVal + if len(callArrayGeneric) >= 3 { + bodyBytes, _ = callArrayGeneric[2].([]byte) + } + } else if nestedArray, ok := callArrayGeneric[1].([]interface{}); ok { + // Nested format: [format, [method, body, ...]] or [format, [{method: "...", body: ...}]] + if len(nestedArray) < 1 { + return info, "nested array format: element[1] is empty array" + } + if methodVal, ok := nestedArray[0].(string); ok { + // Nested array format: [format, [method, body, ...]] + method = methodVal + if len(nestedArray) >= 2 { + bodyBytes, _ = nestedArray[1].([]byte) + } + } else if nestedMap, ok := nestedArray[0].(map[interface{}]interface{}); ok { + // Nested map format: [format, [{method: "...", body: ...}]] + if methodVal, ok := nestedMap["method"].(string); ok { + method = methodVal + if bodyVal, ok := nestedMap["body"].([]byte); ok { + bodyBytes = bodyVal + } + } else { + return info, "nested map format: method field missing or invalid" + } + } else { + return info, fmt.Sprintf("nested array format: element[1][0] is not a valid method string or map (type: %T)", nestedArray[0]) + } + } else { + return info, fmt.Sprintf("array format: element[1] is not a valid method string or array (type: %T)", callArrayGeneric[1]) + } + + info.Method = method + // Decode EVM call methods that have body structure + if bodyBytes != nil && (method == "evm.Call" || method == "evm.Create" || method == "evm.SimulateCall" || method == "evm.EstimateGas") { + info.EVMCall = decodeEVMCallBodyFromCBOR(bodyBytes, method == "evm.Create") + } + return info, "" + } + + // Final fallback: try legacy map[interface{}]interface{} decode var call map[interface{}]interface{} if err := cbor.Unmarshal(ia.Input, &call); err == nil { if methodVal, ok := call["method"].(string); ok { @@ -177,20 +253,22 @@ func decodeEVMTxInput(key []byte, value []byte) *EVMTxInputInfo { // Decode EVM call methods that have body structure if methodVal == "evm.Call" || methodVal == "evm.Create" || methodVal == "evm.SimulateCall" || methodVal == "evm.EstimateGas" { isCreate := methodVal == "evm.Create" - info.EVMCall = decodeEVMCallBody(call, isCreate) + // Extract body bytes and decode + if bodyBytes, ok := call["body"].([]byte); ok { + info.EVMCall = decodeEVMCallBodyFromCBOR(bodyBytes, isCreate) + } } - } else { - info.DecodeError = "method field missing or invalid in call structure" + return info, "" } - } else { - info.DecodeError = fmt.Sprintf("failed to unmarshal call structure: %v", err) + return info, "map format: method field missing or invalid" } - return info + // All decode attempts failed + return info, "all formats failed" } -// decodeEVMCallBody decodes EVM call/create body from SDK Call structure. -func decodeEVMCallBody(call map[interface{}]interface{}, isCreate bool) *EVMCallInfo { +// decodeEVMCallBodyFromCBOR decodes EVM call/create body from raw CBOR bytes. +func decodeEVMCallBodyFromCBOR(bodyBytes []byte, isCreate bool) *EVMCallInfo { info := &EVMCallInfo{} if isCreate { info.Type = "create" @@ -198,11 +276,6 @@ func decodeEVMCallBody(call map[interface{}]interface{}, isCreate bool) *EVMCall info.Type = "call" } - bodyBytes, ok := call["body"].([]byte) - if !ok { - return info - } - var bodyMap map[interface{}]interface{} if err := cbor.Unmarshal(bodyBytes, &bodyMap); err != nil { return info @@ -211,7 +284,7 @@ func decodeEVMCallBody(call map[interface{}]interface{}, isCreate bool) *EVMCall // Extract address (calls only) if !isCreate { if addrBytes, ok := bodyMap["address"].([]byte); ok && len(addrBytes) == 20 { - info.Address = fmt.Sprintf("0x%x", addrBytes) + info.Address = truncateHex0x(addrBytes, TruncateLongSize) } } @@ -227,25 +300,29 @@ func decodeEVMCallBody(call map[interface{}]interface{}, isCreate bool) *EVMCall } if dataBytes, ok := bodyMap[dataKey].([]byte); ok { info.DataSize = len(dataBytes) - info.DataRaw = truncateHex0x(dataBytes, 64) // 32 bytes = 64 hex chars + info.DataHex = truncateHex(dataBytes, TruncateLongSize) } return info } // decodeEVMTxOutput decodes EVM transaction output artifacts. -func decodeEVMTxOutput(value []byte) *EVMTxOutputInfo { - info := &EVMTxOutputInfo{OutputSize: len(value)} +// Returns (info, error) where error is a parsing error. +// Execution errors (transaction failures) are stored in info.Error field. +func decodeEVMTxOutput(value []byte) (*EVMTxOutputInfo, string) { + info := &EVMTxOutputInfo{ + TxOutputSize: len(value), + TxOutputHex: truncateHex(value, TruncateLongSize), + } - var oa RuntimeOutputArtifacts + var oa cborRuntimeOutputArtifacts if err := cbor.Unmarshal(value, &oa); err != nil { - info.DecodeError = err.Error() - return info + return nil, err.Error() } if len(oa.Output) == 0 { info.Success = true - return info + return info, "" } // Try to decode as CBOR CallResult @@ -254,8 +331,8 @@ func decodeEVMTxOutput(value []byte) *EVMTxOutputInfo { // Raw bytes - success info.Success = true info.ResultSize = len(oa.Output) - info.ResultRaw = truncateHex0x(oa.Output, 64) // 32 bytes = 64 hex chars - return info + info.ResultHex = truncateHex(oa.Output, TruncateLongSize) + return info, "" } // Check success/failure @@ -263,18 +340,18 @@ func decodeEVMTxOutput(value []byte) *EVMTxOutputInfo { info.Success = true if okBytes, ok := okVal.([]byte); ok { info.ResultSize = len(okBytes) - info.ResultRaw = truncateHex0x(okBytes, 64) + info.ResultHex = truncateHex(okBytes, TruncateLongSize) } } else if failVal, exists := result["fail"]; exists { info.Success = false if failMap, ok := failVal.(map[interface{}]interface{}); ok { if msgVal, ok := failMap["message"].(string); ok { - info.Error = msgVal + info.Error = msgVal // Execution error, keep in struct } else { info.Error = fmt.Sprintf("%v", failMap) } } } - return info + return info, "" } diff --git a/badgerdb-sampler/decode_runtime.go b/badgerdb-sampler/decode_runtime.go index 3fc062a..2fe5e38 100644 --- a/badgerdb-sampler/decode_runtime.go +++ b/badgerdb-sampler/decode_runtime.go @@ -8,15 +8,14 @@ import ( "github.com/fxamacker/cbor/v2" ) -// decodeKeyRuntimeMkvs parses runtime-mkvs key and returns structured info. +// decodeRuntimeMkvsKey parses runtime-mkvs key and returns structured info. // Keys use keyformat encoding: [type_byte][data...] // See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 -func decodeKeyRuntimeMkvs(key []byte) RuntimeMkvsKeyInfo { +func decodeRuntimeMkvsKey(key []byte) RuntimeMkvsKeyInfo { info := RuntimeMkvsKeyInfo{} if len(key) < 1 { info.KeyType = "unknown" - info.DecodeError = "key too short" return info } @@ -27,42 +26,35 @@ func decodeKeyRuntimeMkvs(key []byte) RuntimeMkvsKeyInfo { switch prefixByte { case 0x00: info.KeyType = "node" - info.Hash = truncateHex(data, 16) + info.Hash = truncateHex(data, TruncateHashSize) case 0x01: info.KeyType = "write_log" if len(data) >= 8 { info.RuntimeHeight = binary.BigEndian.Uint64(data[0:8]) - } else { - info.DecodeError = "write_log data too short" } case 0x02: info.KeyType = "roots_metadata" if len(data) >= 8 { info.RuntimeHeight = binary.BigEndian.Uint64(data[0:8]) - } else { - info.DecodeError = "roots_metadata data too short" } case 0x03: info.KeyType = "root_updated_nodes" if len(data) >= 8 { info.RuntimeHeight = binary.BigEndian.Uint64(data[0:8]) - } else { - info.DecodeError = "root_updated_nodes data too short" } case 0x04: info.KeyType = "metadata" default: info.KeyType = fmt.Sprintf("unknown_%02x", prefixByte) - info.DecodeError = fmt.Sprintf("unknown key prefix 0x%02x", prefixByte) } return info } -// decodeValueRuntimeMkvs decodes runtime MKVS value and returns structured info. +// decodeRuntimeMkvsValue decodes runtime MKVS value and returns structured info. // See: _oasis-core/go/storage/mkvs/node/node.go:26-32 (prefixes), 294-309 (InternalNode), 531-537 (LeafNode) -func decodeValueRuntimeMkvs(keyType string, value []byte) RuntimeMkvsNodeInfo { - info := RuntimeMkvsNodeInfo{Size: len(value)} +func decodeRuntimeMkvsValue(keyType string, value []byte) RuntimeMkvsNodeInfo { + info := RuntimeMkvsNodeInfo{NodeSize: len(value)} if len(value) == 0 { info.NodeType = "empty" @@ -74,55 +66,59 @@ func decodeValueRuntimeMkvs(keyType string, value []byte) RuntimeMkvsNodeInfo { return info } + // Add raw node hex dump + info.NodeHex = truncateHex(value, TruncateLongSize) + // Parse MKVS node: 0x00=leaf, 0x01=internal, 0x02=nil switch value[0] { - case 0x00: // LeafNode: [2-byte keyLen LE][key][4-byte valueLen LE][value] + case 0x00: // LeafNode: [2-byte keySize LE][key][4-byte valueSize LE][value] info.NodeType = "leaf" data := value[1:] if len(data) < 2 { - info.DecodeError = "key length missing" + info.LeafError = "key length missing" return info } - keyLen := int(binary.LittleEndian.Uint16(data[0:2])) + keySize := int(binary.LittleEndian.Uint16(data[0:2])) data = data[2:] - if len(data) < keyLen { - info.DecodeError = fmt.Sprintf("key truncated (expected %d bytes)", keyLen) + if len(data) < keySize { + info.LeafError = fmt.Sprintf("key truncated (expected %d bytes)", keySize) return info } - key := data[:keyLen] - data = data[keyLen:] + key := data[:keySize] + data = data[keySize:] // Extract module name from key module := extractModuleName(key) leaf := &RuntimeMkvsLeafInfo{ - Module: module, - KeyLen: keyLen, - Key: truncateHex(key, 32), + Module: module, + KeySize: keySize, + KeyHex: truncateHex(key, TruncateLongSize), } if len(data) < 4 { - info.DecodeError = "value length missing" + info.LeafError = "value length missing" info.Leaf = leaf return info } - valueLen := int(binary.LittleEndian.Uint32(data[0:4])) + valueSize := int(binary.LittleEndian.Uint32(data[0:4])) data = data[4:] - leaf.ValueLen = valueLen + leaf.ValueSize = valueSize - if len(data) < valueLen { - info.DecodeError = fmt.Sprintf("value truncated (expected %d bytes)", valueLen) + if len(data) < valueSize { + info.LeafError = fmt.Sprintf("value truncated (expected %d bytes)", valueSize) info.Leaf = leaf return info } - leafValue := data[:valueLen] - leaf.DecodedValue = decodeLeafValue(module, key, leafValue) + leafValue := data[:valueSize] + leaf.ValueHex = truncateHex(leafValue, TruncateLongSize) + leaf.Value = decodeRuntimeLeafValue(module, key, leafValue) info.Leaf = leaf return info @@ -131,7 +127,7 @@ func decodeValueRuntimeMkvs(keyType string, value []byte) RuntimeMkvsNodeInfo { data := value[1:] if len(data) < 2 { - info.DecodeError = "label bits missing" + info.InternalError = "label bits missing" return info } @@ -140,7 +136,7 @@ func decodeValueRuntimeMkvs(keyType string, value []byte) RuntimeMkvsNodeInfo { labelBytes := (int(labelBits) + 7) / 8 if len(data) < labelBytes+1 { - info.DecodeError = "label truncated" + info.InternalError = "label truncated" info.Internal = &RuntimeMkvsInternalInfo{LabelBits: labelBits} return info } @@ -155,11 +151,11 @@ func decodeValueRuntimeMkvs(keyType string, value []byte) RuntimeMkvsNodeInfo { data = data[1:] // skip nil marker // Extract child hashes if len(data) >= 32 { - internal.LeftHash = truncateHex(data[:32], 16) + internal.LeftHash = truncateHex(data[:32], TruncateHashSize) data = data[32:] } if len(data) >= 32 { - internal.RightHash = truncateHex(data[:32], 16) + internal.RightHash = truncateHex(data[:32], TruncateHashSize) } } @@ -172,23 +168,22 @@ func decodeValueRuntimeMkvs(keyType string, value []byte) RuntimeMkvsNodeInfo { default: info.NodeType = "unknown" - info.DecodeError = fmt.Sprintf("unknown prefix 0x%02x", value[0]) + info.NodeError = fmt.Sprintf("unknown prefix 0x%02x", value[0]) return info } } -// decodeKeyRuntimeHistory parses runtime-history key and returns structured info. +// decodeRuntimeHistoryKey parses runtime-history key and returns structured info. // See: _oasis-core/go/runtime/history/db.go:19-31 // Key formats: // - 0x01: metadata key (1 byte) // - 0x02 + uint64: block key (9 bytes) - round number in big-endian // - 0x03 + uint64: round_results key (9 bytes) - round number in big-endian -func decodeKeyRuntimeHistory(key []byte) RuntimeHistoryKeyInfo { +func decodeRuntimeHistoryKey(key []byte) RuntimeHistoryKeyInfo { info := RuntimeHistoryKeyInfo{} if len(key) == 0 { info.KeyType = "unknown" - info.DecodeError = "empty key" return info } @@ -196,66 +191,66 @@ func decodeKeyRuntimeHistory(key []byte) RuntimeHistoryKeyInfo { case 0x01: info.KeyType = "metadata" if len(key) > 1 { - info.ExtraData = fmt.Sprintf("%x", key[1:]) + info.ExtraData = truncateHex(key[1:], TruncateLongSize) } case 0x02: info.KeyType = "block" if len(key) == 9 { info.RuntimeHeight = binary.BigEndian.Uint64(key[1:9]) - } else { - info.DecodeError = "block key wrong length" } case 0x03: info.KeyType = "round_results" if len(key) == 9 { info.RuntimeHeight = binary.BigEndian.Uint64(key[1:9]) - } else { - info.DecodeError = "round_results key wrong length" } default: info.KeyType = "unknown" - info.DecodeError = fmt.Sprintf("unknown key prefix 0x%02x", key[0]) } return info } -// decodeValueRuntimeHistory decodes CBOR-encoded runtime history value and returns structured info. +// decodeRuntimeHistoryValue decodes CBOR-encoded runtime history value and returns structured info. // See: _oasis-core/go/runtime/history/db.go:34-44 (metadata) // See: _oasis-core/go/roothash/api/api.go:402-409 (AnnotatedBlock) // See: _oasis-core/go/roothash/api/results.go:5-17 (RoundResults) -func decodeValueRuntimeHistory(keyType string, value []byte) RuntimeHistoryValueInfo { - info := RuntimeHistoryValueInfo{ - KeyType: keyType, - Size: len(value), - } +func decodeRuntimeHistoryValue(keyType string, value []byte) RuntimeHistoryValueInfo { + info := RuntimeHistoryValueInfo{} if len(value) == 0 { - info.DecodeError = "empty value" + // General error when we don't know which child yet + switch keyType { + case "metadata": + info.MetadataError = "empty value" + case "block": + info.BlockError = "empty value" + case "round_results": + info.RoundResultsError = "empty value" + } return info } switch keyType { case "metadata": - var meta RuntimeHistoryMetadata + var meta cborRuntimeHistoryMetadata if err := cbor.Unmarshal(value, &meta); err != nil { - info.DecodeError = err.Error() + info.MetadataError = err.Error() return info } info.Metadata = &RuntimeHistoryMetadataInfo{ - Version: meta.Version, - RuntimeID: fmt.Sprintf("%x", meta.RuntimeID), - LastRuntimeHeight: meta.LastRound, - LastConsensusHeight: meta.LastConsensusHeight, + Version: meta.Version, + RuntimeID: truncateHex(meta.RuntimeID, TruncateLongSize), + LastRuntimeHeight: meta.LastRound, + LastConsensusHeight: meta.LastConsensusHeight, } case "block": - var block RuntimeHistoryAnnotatedBlock + var block cborRuntimeHistoryAnnotatedBlock if err := cbor.Unmarshal(value, &block); err != nil { - info.DecodeError = err.Error() + info.BlockError = err.Error() return info } blockInfo := &RuntimeHistoryBlockInfo{ @@ -268,14 +263,14 @@ func decodeValueRuntimeHistory(keyType string, value []byte) RuntimeHistoryValue blockInfo.RuntimeHeight = h.Round blockInfo.Timestamp = time.Unix(int64(h.Timestamp), 0).UTC().Format(time.RFC3339) blockInfo.HeaderType = headerTypeName(h.HeaderType) - blockInfo.StateRoot = truncateHex(h.StateRoot, 32) + blockInfo.StateRoot = truncateHex(h.StateRoot, TruncateHashSize) } info.Block = blockInfo case "round_results": - var results RuntimeHistoryRoundResults + var results cborRuntimeHistoryRoundResults if err := cbor.Unmarshal(value, &results); err != nil { - info.DecodeError = err.Error() + info.RoundResultsError = err.Error() return info } info.RoundResults = &RuntimeHistoryRoundResultsInfo{ @@ -283,9 +278,6 @@ func decodeValueRuntimeHistory(keyType string, value []byte) RuntimeHistoryValue GoodComputeEntities: len(results.GoodComputeEntities), BadComputeEntities: len(results.BadComputeEntities), } - - default: - info.DecodeError = fmt.Sprintf("unknown key type: %s", keyType) } return info @@ -309,16 +301,18 @@ func headerTypeName(headerType uint8) string { } } -// decodeLeafValue decodes MKVS leaf value and returns structured info. +// decodeRuntimeLeafValue decodes MKVS leaf value and returns structured info with valueType classification. // See: _oasis-core/go/runtime/transaction/transaction.go:129-150 (artifacts) -func decodeLeafValue(module string, key []byte, value []byte) *RuntimeLeafValueInfo { +func decodeRuntimeLeafValue(module string, key []byte, value []byte) *RuntimeLeafValueInfo { info := &RuntimeLeafValueInfo{} + info.ValueType = "binary" // default // Check for EVM module data if len(module) >= 3 && module[:3] == "evm" { evmInfo := decodeEVMData(module, key, value) if evmInfo != nil { info.EVM = evmInfo + // No error return from decodeEVMData (returns nil on failure) // Set value_type based on EVM storage type switch evmInfo.StorageType { case "code": @@ -341,13 +335,13 @@ func decodeLeafValue(module string, key []byte, value []byte) *RuntimeLeafValueI kind := key[33] if kind == 1 { // Input artifact + info.EVMTxInput, info.EVMTxInputError = decodeEVMTxInput(key, value) info.ValueType = "io_input" - info.EVMTxInput = decodeEVMTxInput(key, value) return info } else if kind == 2 { // Output artifact + info.EVMTxOutput, info.EVMTxOutputError = decodeEVMTxOutput(value) info.ValueType = "io_output" - info.EVMTxOutput = decodeEVMTxOutput(value) return info } } @@ -357,35 +351,31 @@ func decodeLeafValue(module string, key []byte, value []byte) *RuntimeLeafValueI if len(module) > 8 && module[:8] == "io_event" { // Special handling for EVM events if len(module) >= 12 && module[9:12] == "evm" { - evmEvent := decodeEVMEvent(value) - if evmEvent != nil { - info.ValueType = "evm_event" - info.EVMEvent = evmEvent - return info - } + info.EVMEvent, info.EVMEventError = decodeEVMEvent(value) + info.ValueType = "evm_event" + return info } // Generic event decoding for non-EVM events var decoded interface{} if err := cbor.Unmarshal(value, &decoded); err == nil { + // Use detailed formatter to expand arrays and maps + info.CBOR = formatCBORDetailed(decoded) info.ValueType = "io_event" - info.DecodedValue = formatCBOR(decoded, len(value)) - return info + } else { + info.CBORError = err.Error() } - info.ValueType = "binary" - info.BinarySize = len(value) return info } // Try CBOR decode for regular state keys var decoded interface{} if err := cbor.Unmarshal(value, &decoded); err == nil { + info.CBOR = formatCBOR(decoded, len(value)) info.ValueType = "cbor" - info.DecodedValue = formatCBOR(decoded, len(value)) - return info + } else { + info.CBORError = err.Error() } - info.ValueType = "binary" - info.BinarySize = len(value) return info } diff --git a/badgerdb-sampler/main.go b/badgerdb-sampler/main.go index 6f2ce85..87a1e84 100644 --- a/badgerdb-sampler/main.go +++ b/badgerdb-sampler/main.go @@ -16,14 +16,16 @@ var BadgerVersion string // Sample contains both raw and decoded key/value information type Sample struct { - RawKey string `json:"raw_key"` - RawKeySize int `json:"raw_key_size"` - KeyType string `json:"key_type"` - DecodedKey interface{} `json:"decoded_key"` - RawValue string `json:"raw_value"` - RawValueSize int `json:"raw_value_size"` - DecodedValue interface{} `json:"decoded_value"` - Timestamp int64 `json:"timestamp,omitempty"` + KeyRaw string `json:"key_raw"` + KeySize int `json:"key_size"` + KeyType string `json:"key_type"` + Key interface{} `json:"key"` // Decoded key (renamed from Key) + KeyError string `json:"key_error,omitempty"` // Error decoding key + ValueRaw string `json:"value_raw"` + ValueSize int `json:"value_size"` + Value interface{} `json:"value"` // Decoded value (renamed from Value) + ValueError string `json:"value_error,omitempty"` // Error decoding value + Timestamp int64 `json:"timestamp,omitempty"` } // DBStats contains database statistics and samples @@ -172,67 +174,69 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { key := item.Key() sample := Sample{ - RawKey: fmt.Sprintf("%x", key), - RawKeySize: len(key), + KeyRaw: truncateHex(key, TruncateLongSize), + KeySize: len(key), KeyType: "unknown", - DecodedKey: nil, + Key: nil, } // Decode key based on database type switch stats.DatabaseType { case "consensus-blockstore": - keyInfo := decodeKeyConsensusBlockstore(key) + keyInfo := decodeConsensusBlockstoreKey(key) sample.KeyType = keyInfo.KeyType - sample.DecodedKey = keyInfo + sample.Key = keyInfo case "consensus-evidence": - keyInfo := decodeKeyConsensusEvidence(key) + keyInfo := decodeConsensusEvidenceKey(key) sample.KeyType = keyInfo.KeyType - sample.DecodedKey = keyInfo + sample.Key = keyInfo case "consensus-mkvs": - keyInfo := decodeKeyConsensusMkvs(key) + keyInfo := decodeConsensusMkvsKey(key) sample.KeyType = keyInfo.KeyType - sample.DecodedKey = keyInfo + sample.Key = keyInfo case "consensus-state": - keyInfo := decodeKeyConsensusState(key) + keyInfo := decodeConsensusStateKey(key) sample.KeyType = keyInfo.KeyType - sample.DecodedKey = keyInfo + sample.Key = keyInfo case "runtime-mkvs": - keyInfo := decodeKeyRuntimeMkvs(key) + keyInfo := decodeRuntimeMkvsKey(key) sample.KeyType = keyInfo.KeyType - sample.DecodedKey = keyInfo + sample.Key = keyInfo case "runtime-history": - keyInfo := decodeKeyRuntimeHistory(key) + keyInfo := decodeRuntimeHistoryKey(key) sample.KeyType = keyInfo.KeyType - sample.DecodedKey = keyInfo + sample.Key = keyInfo } // Fetch and decode value err := item.Value(func(val []byte) error { - sample.RawValueSize = len(val) - sample.RawValue = truncateHex(val, 200) - sample.DecodedValue = nil + sample.ValueSize = len(val) + sample.ValueRaw = truncateHex(val, TruncateLongSize) + sample.Value = nil // Decode value based on database type switch stats.DatabaseType { case "consensus-blockstore": - valueInfo := decodeValueConsensusBlockstore(sample.KeyType, val) - sample.DecodedValue = valueInfo + valueInfo := decodeConsensusBlockstoreValue(sample.KeyType, val) + sample.Value = valueInfo sample.Timestamp = valueInfo.Timestamp case "consensus-evidence": - sample.DecodedValue = decodeValueConsensusEvidence(sample.KeyType, val) + value, err := decodeConsensusEvidenceValue(sample.KeyType, val) + sample.Value = value + sample.ValueError = err case "consensus-mkvs": - sample.DecodedValue = decodeValueConsensusMkvs(sample.KeyType, val) + sample.Value = decodeConsensusMkvsValue(sample.KeyType, val) case "consensus-state": - sample.DecodedValue = decodeValueConsensusState(sample.KeyType, val) + sample.Value = decodeConsensusStateValue(sample.KeyType, val) case "runtime-mkvs": - sample.DecodedValue = decodeValueRuntimeMkvs(sample.KeyType, val) + sample.Value = decodeRuntimeMkvsValue(sample.KeyType, val) case "runtime-history": - sample.DecodedValue = decodeValueRuntimeHistory(sample.KeyType, val) + sample.Value = decodeRuntimeHistoryValue(sample.KeyType, val) } return nil }) if err != nil { - log.Printf("Error reading value for key %s: %v", sample.RawKey, err) + log.Printf("Error reading value for key %s: %v", sample.KeyRaw, err) } stats.Samples = append(stats.Samples, sample) diff --git a/badgerdb-sampler/types_consensus.go b/badgerdb-sampler/types_consensus.go index fcf66b6..6fe8cc8 100644 --- a/badgerdb-sampler/types_consensus.go +++ b/badgerdb-sampler/types_consensus.go @@ -1,6 +1,10 @@ package main -import "time" +import ( + "time" + + "github.com/fxamacker/cbor/v2" +) // Consensus decode output types - structured representations of decoded consensus database entries. // These types separate decoding logic from string formatting, enabling flexible output formats. @@ -16,22 +20,24 @@ type ConsensusBlockstoreKeyInfo struct { ConsensusHeight int64 `json:"consensus_height,omitempty"` // For H:, C:, SC:, P: keys PartIndex int `json:"part_index,omitempty"` // For P: keys Hash string `json:"hash,omitempty"` // For BH: keys - RawKey string `json:"raw_key,omitempty"` // Original ASCII key + KeySize int `json:"key_size"` + KeyRaw string `json:"key_raw,omitempty"` // Original ASCII key } // ConsensusBlockstoreValueInfo represents a decoded consensus-blockstore value. // See: tendermint/proto/tendermint/store/types.proto (BlockStoreState) // See: tendermint/proto/tendermint/types/types.proto (BlockMeta, Part, Commit) type ConsensusBlockstoreValueInfo struct { - KeyType string `json:"key_type"` - Size int `json:"size"` - Timestamp int64 `json:"timestamp,omitempty"` // Unix timestamp from block meta - State *ConsensusBlockStoreState `json:"state,omitempty"` - BlockMeta *ConsensusBlockMetaInfo `json:"block_meta,omitempty"` - Part *ConsensusPartInfo `json:"part,omitempty"` - Commit *ConsensusCommitInfo `json:"commit,omitempty"` - HashHeight int64 `json:"hash_height,omitempty"` // For block_hash type - DecodeError string `json:"decode_error,omitempty"` + Timestamp int64 `json:"timestamp,omitempty"` // Unix timestamp from block meta + State *ConsensusBlockStoreState `json:"state,omitempty"` + StateError string `json:"state_error,omitempty"` // Error decoding state + BlockMeta *ConsensusBlockMetaInfo `json:"block_meta,omitempty"` + BlockMetaError string `json:"block_meta_error,omitempty"` // Error decoding block meta + Part *ConsensusPartInfo `json:"part,omitempty"` + PartError string `json:"part_error,omitempty"` // Error decoding part + Commit *ConsensusCommitInfo `json:"commit,omitempty"` + CommitError string `json:"commit_error,omitempty"` // Error decoding commit + HashHeight int64 `json:"hash_height,omitempty"` // For block_hash type } // ConsensusBlockStoreState represents BlockStoreState from tendermint. @@ -55,7 +61,7 @@ type ConsensusBlockMetaInfo struct { // See: tendermint/proto/tendermint/types/types.proto (Part message) type ConsensusPartInfo struct { Index uint32 `json:"index"` - BytesSize int `json:"bytes_size"` + PartSize int `json:"part_size"` ProofTotal int64 `json:"proof_total"` } @@ -76,21 +82,21 @@ type ConsensusCommitInfo struct { type ConsensusEvidenceKeyInfo struct { KeyType string `json:"key_type"` PrefixByte byte `json:"prefix_byte,omitempty"` - RawKey string `json:"raw_key,omitempty"` // hex + KeySize int `json:"key_size"` + KeyHex string `json:"key_hex,omitempty"` // hex } // ConsensusEvidenceValueInfo represents a decoded consensus-evidence value. // See: tendermint/proto/tendermint/types/evidence.proto (DuplicateVoteEvidence) type ConsensusEvidenceValueInfo struct { - Size int `json:"size"` - VoteAHeight int64 `json:"vote_a_height,omitempty"` - VoteBHeight int64 `json:"vote_b_height,omitempty"` - EvidenceType string `json:"evidence_type,omitempty"` // "duplicate_vote", "light_client_attack", "unknown" - TotalVotingPower int64 `json:"total_voting_power,omitempty"` - ValidatorPower int64 `json:"validator_power,omitempty"` - Timestamp string `json:"timestamp,omitempty"` // RFC3339 format - RawValue string `json:"raw_value,omitempty"` // First 32 hex bytes if decode fails - DecodeError string `json:"decode_error,omitempty"` + ValueSize int `json:"value_size"` + ValueHex string `json:"value_hex,omitempty"` // Truncated hex dump + VoteAHeight int64 `json:"vote_a_height,omitempty"` + VoteBHeight int64 `json:"vote_b_height,omitempty"` + EvidenceType string `json:"evidence_type,omitempty"` // "duplicate_vote", "light_client_attack", "unknown" + TotalVotingPower int64 `json:"total_voting_power,omitempty"` + ValidatorPower int64 `json:"validator_power,omitempty"` + Timestamp string `json:"timestamp,omitempty"` // RFC3339 format } // ============================================================================= @@ -100,21 +106,25 @@ type ConsensusEvidenceValueInfo struct { // ConsensusMkvsKeyInfo represents a decoded consensus-mkvs key. // See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 type ConsensusMkvsKeyInfo struct { + KeySize int `json:"key_size"` + KeyHex string `json:"key_hex,omitempty"` // hex, truncated KeyType string `json:"key_type"` // "node", "write_log", "roots_metadata", "root_updated_nodes", "metadata", "multipart_restore_log", "root_node", "unknown" ConsensusHeight uint64 `json:"consensus_height,omitempty"` // For write_log, roots_metadata, root_updated_nodes - Hash string `json:"hash,omitempty"` // hex, truncated + Hash string `json:"hash,omitempty"` // hex, truncated (partial key data) RootType byte `json:"root_type,omitempty"` - DecodeError string `json:"decode_error,omitempty"` } -// ConsensusMkvsNodeInfo represents a decoded consensus-mkvs value (node). +// ConsensusMkvsValueInfo represents a decoded consensus-mkvs value (node). // See: _oasis-core/go/storage/mkvs/node/node.go:26-32 (prefixes), 294-309 (InternalNode), 531-537 (LeafNode) -type ConsensusMkvsNodeInfo struct { - NodeType string `json:"node_type"` // "leaf", "internal", "nil", "non_node", "unknown" - Size int `json:"size"` - Leaf *ConsensusMkvsLeafInfo `json:"leaf,omitempty"` - Internal *ConsensusMkvsInternalInfo `json:"internal,omitempty"` - DecodeError string `json:"decode_error,omitempty"` +type ConsensusMkvsValueInfo struct { + NodeType string `json:"node_type"` // "leaf", "internal", "nil", "non_node", "unknown" + NodeSize int `json:"node_size"` // Total MKVS node size + NodeHex string `json:"node_hex,omitempty"` // Raw node dump (hex, truncated) + Leaf *ConsensusMkvsLeafInfo `json:"leaf,omitempty"` + LeafError string `json:"leaf_error,omitempty"` // Error decoding leaf node + Internal *ConsensusMkvsInternalInfo `json:"internal,omitempty"` + InternalError string `json:"internal_error,omitempty"` // Error decoding internal node + NodeError string `json:"node_error,omitempty"` // Structural parsing error } // ConsensusMkvsLeafInfo represents a decoded MKVS LeafNode for consensus. @@ -122,11 +132,12 @@ type ConsensusMkvsNodeInfo struct { // See: _oasis-core/go/consensus/tendermint/apps/*/state/state.go (module prefixes) type ConsensusMkvsLeafInfo struct { Module string `json:"module"` - KeyLen int `json:"key_len"` - Key string `json:"key,omitempty"` // hex, truncated + KeySize int `json:"key_size"` + KeyHex string `json:"key_hex,omitempty"` // hex, truncated OasisAddress string `json:"oasis_address,omitempty"` // bech32 oasis1... address if applicable - ValueLen int `json:"value_len"` - DecodedValue interface{} `json:"decoded_value,omitempty"` // CBOR decoded or size info + ValueSize int `json:"value_size"` + ValueHex string `json:"value_hex,omitempty"` // hex, truncated + ValueFormatted interface{} `json:"value_formatted,omitempty"` // CBOR formatted output from formatCBOR() } // ConsensusMkvsInternalInfo represents a decoded MKVS InternalNode. @@ -146,7 +157,8 @@ type ConsensusMkvsInternalInfo struct { // See: tendermint/state/store.go for key formats (abciResponsesKey, validatorsKey, etc.) type ConsensusStateKeyInfo struct { KeyType string `json:"key_type"` // "abci_responses", "consensus_params", "validators", "state", "genesis", "text_key", "binary", "unknown" - DecodedKey string `json:"decoded_key,omitempty"` + KeySize int `json:"key_size"` + KeyHex string `json:"key_hex,omitempty"` // hex, truncated ConsensusHeight int64 `json:"consensus_height,omitempty"` // For abci_responses (extracted from key) IsBinary bool `json:"is_binary,omitempty"` } @@ -154,19 +166,21 @@ type ConsensusStateKeyInfo struct { // ConsensusStateValueInfo represents a decoded consensus-state value. // See: tendermint/proto/tendermint/state/types.proto (various state types) type ConsensusStateValueInfo struct { - KeyType string `json:"key_type"` - Size int `json:"size"` - ABCIInfo *ABCIResponseInfo `json:"abci_info,omitempty"` // For abci_responses - ConsensusParams *tmConsensusParams `json:"consensus_params,omitempty"` // For consensus_params - ConsensusValidators *tmValidatorSet `json:"consensus_validators,omitempty"` // For validators - ConsensusState *tmState `json:"consensus_state,omitempty"` // For state - RawValue string `json:"raw_value,omitempty"` // Raw hex dump - DecodeError string `json:"decode_error,omitempty"` -} - -// ABCIResponseInfo represents decoded ABCI response summary. + ValueSize int `json:"value_size"` + ValueHex string `json:"value_hex,omitempty"` // Raw hex dump + ABCIResponse *ConsensusABCIResponseInfo `json:"abci_response,omitempty"` // For abci_responses + ABCIResponseError string `json:"abci_response_error,omitempty"` // Error decoding ABCI responses + ConsensusParams *tmConsensusParams `json:"consensus_params,omitempty"` // For consensus_params + ConsensusParamsError string `json:"consensus_params_error,omitempty"` // Error decoding consensus params + ConsensusValidators *tmValidatorSet `json:"consensus_validators,omitempty"` // For validators + ValidatorsError string `json:"validators_error,omitempty"` // Error decoding validators + ConsensusState *tmState `json:"consensus_state,omitempty"` // For state + StateError string `json:"state_error,omitempty"` // Error decoding state +} + +// ConsensusABCIResponseInfo represents decoded ABCI response summary. // See: tendermint/proto/tendermint/abci/types.proto (ResponseFinalizeBlock) -type ABCIResponseInfo struct { +type ConsensusABCIResponseInfo struct { TxResultCount int `json:"tx_result_count"` ValidatorUpdates int `json:"validator_updates"` EventCount int `json:"event_count"` @@ -176,22 +190,13 @@ type ABCIResponseInfo struct { // ConsensusTransactionSummary summarizes decoded transactions from ABCI responses. type ConsensusTransactionSummary struct { - MethodCounts map[string]int `json:"method_counts"` // Count by method name - Transactions []ConsensusDecodedTransaction `json:"transactions,omitempty"` // Decoded transaction details + MethodCounts map[string]int `json:"method_counts"` // Count by method name + Transactions []ConsensusTransactionInfo `json:"transactions,omitempty"` // Decoded transaction details } -// ConsensusEventSummary summarizes decoded events from ABCI responses. -type ConsensusEventSummary struct { - EventTypeCounts map[string]int `json:"event_type_counts"` // Count by event type -} - -// ============================================================================= -// Transaction and Event Types (Oasis-specific) -// ============================================================================= - -// ConsensusDecodedTransaction represents a decoded Oasis consensus transaction. +// ConsensusTransactionInfo represents a decoded Oasis consensus transaction. // See: _oasis-core/go/consensus/api/transaction/transaction.go:42-54 -type ConsensusDecodedTransaction struct { +type ConsensusTransactionInfo struct { Nonce uint64 `json:"nonce"` Method string `json:"method"` Fee *ConsensusFee `json:"fee,omitempty"` @@ -199,7 +204,6 @@ type ConsensusDecodedTransaction struct { TxHash string `json:"tx_hash,omitempty"` // hex-encoded transaction hash BodyPreview string `json:"body_preview,omitempty"` // hex preview if body not decoded DecodedBody interface{} `json:"decoded_body,omitempty"` // Decoded body for known types - DecodeError string `json:"decode_error,omitempty"` } // ConsensusFee represents transaction fee. @@ -209,125 +213,144 @@ type ConsensusFee struct { Gas uint64 `json:"gas"` } -// ConsensusSignedTransaction represents the signature envelope. +// ConsensusEventSummary summarizes decoded events from ABCI responses. +type ConsensusEventSummary struct { + EventTypeCounts map[string]int `json:"event_type_counts"` // Count by event type +} + + +// ============================================================================= +// Consensus CBOR Deserialization Types +// ============================================================================= + +// cborConsensusSignedTransaction represents the signature envelope. // See: _oasis-core/go/common/crypto/signature/signature.go:415-421 -type ConsensusSignedTransaction struct { - Blob []byte `json:"untrusted_raw_value"` - Signature ConsensusSignature `json:"signature"` +type cborConsensusSignedTransaction struct { + Blob []byte `json:"untrusted_raw_value"` + Signature cborConsensusSignature `json:"signature"` +} + +// cborConsensusInnerTransaction represents an unsigned consensus transaction. +// See: _oasis-core/go/consensus/api/transaction/transaction.go:43-54 +type cborConsensusInnerTransaction struct { + Nonce uint64 + Fee *cborConsensusFee + Method string + Body cbor.RawMessage } -// ConsensusSignature represents a signature with public key. +// cborConsensusFee represents transaction fee. +// See: _oasis-core/go/consensus/api/transaction/gas.go:30-35 +type cborConsensusFee struct { + Amount string `json:"amount"` // string representation of quantity.Quantity + Gas uint64 `json:"gas"` +} + +// cborConsensusSignature represents a signature with public key. // See: _oasis-core/go/common/crypto/signature/signature.go:313-318 -type ConsensusSignature struct { +type cborConsensusSignature struct { PublicKey [32]byte `json:"public_key"` // ED25519 public key Signature [64]byte `json:"signature"` // ED25519 signature } -// ============================================================================= -// Transaction Body Types (Staking Module) -// ============================================================================= - -// ConsensusTransfer represents a staking transfer transaction body. +// cborConsensusTransfer represents a staking transfer transaction body. // See: _oasis-core/go/staking/api/api.go:342-345 -type ConsensusTransfer struct { +type cborConsensusTransfer struct { To [21]byte `json:"to"` // Oasis address Amount string `json:"amount"` // string representation of quantity.Quantity } -// ConsensusBurn represents a staking burn transaction body. +// cborConsensusBurn represents a staking burn transaction body. // See: _oasis-core/go/staking/api/api.go:368-370 -type ConsensusBurn struct { +type cborConsensusBurn struct { Amount string `json:"amount"` // string representation of quantity.Quantity } -// ConsensusAddEscrow represents an add escrow transaction body. +// cborConsensusAddEscrow represents an add escrow transaction body. // See: _oasis-core/go/staking/api/api.go:403-406 -type ConsensusAddEscrow struct { +type cborConsensusAddEscrow struct { Account [21]byte `json:"account"` // Oasis address Amount string `json:"amount"` // string representation of quantity.Quantity } -// ConsensusReclaimEscrow represents a reclaim escrow transaction body. +// cborConsensusReclaimEscrow represents a reclaim escrow transaction body. // See: _oasis-core/go/staking/api/api.go:444-447 -type ConsensusReclaimEscrow struct { +type cborConsensusReclaimEscrow struct { Account [21]byte `json:"account"` // Oasis address Shares string `json:"shares"` // string representation of quantity.Quantity } -// ConsensusRegisterEntity represents a registry entity registration transaction body. +// cborConsensusRegisterEntity represents a registry entity registration transaction body. // See: _oasis-core/go/registry/api/api.go -type ConsensusRegisterEntity struct { +type cborConsensusRegisterEntity struct { Signature []byte `json:"signature,omitempty"` Descriptor []byte `json:"descriptor,omitempty"` // CBOR-encoded entity descriptor } -// ConsensusRegisterNode represents a registry node registration transaction body. +// cborConsensusRegisterNode represents a registry node registration transaction body. // See: _oasis-core/go/registry/api/api.go -type ConsensusRegisterNode struct { +type cborConsensusRegisterNode struct { Signature []byte `json:"signature,omitempty"` Descriptor []byte `json:"descriptor,omitempty"` // CBOR-encoded node descriptor } -// ConsensusExecutorCommit represents a roothash executor commit transaction body. +// cborConsensusExecutorCommit represents a roothash executor commit transaction body. // See: _oasis-core/go/roothash/api/commitment/executor.go -type ConsensusExecutorCommit struct { +type cborConsensusExecutorCommit struct { ID []byte `json:"runtime_id,omitempty"` // Runtime ID Commits []byte `json:"commits,omitempty"` // CBOR-encoded commits } -// ConsensusSubmitProposal represents a governance proposal submission transaction body. +// cborConsensusSubmitProposal represents a governance proposal submission transaction body. // See: _oasis-core/go/governance/api/api.go -type ConsensusSubmitProposal struct { +type cborConsensusSubmitProposal struct { Content []byte `json:"content,omitempty"` // CBOR-encoded proposal content Deposit string `json:"deposit,omitempty"` // string representation of quantity.Quantity } -// ConsensusCastVote represents a governance vote transaction body. +// cborConsensusCastVote represents a governance vote transaction body. // See: _oasis-core/go/governance/api/api.go -type ConsensusCastVote struct { +type cborConsensusCastVote struct { ProposalID uint64 `json:"proposal_id"` Vote uint8 `json:"vote"` // 0=invalid, 1=yes, 2=no, 3=abstain } -// ============================================================================= -// Event Types (Staking Module) -// ============================================================================= - -// ConsensusTransferEvent represents a staking transfer event. +// cborConsensusTransferEvent represents a staking transfer event. // See: _oasis-core/go/staking/api/api.go:223-229 -type ConsensusTransferEvent struct { +type cborConsensusTransferEvent struct { From [21]byte `json:"from"` // Oasis address To [21]byte `json:"to"` // Oasis address Amount string `json:"amount"` // string representation of quantity.Quantity } -// ConsensusBurnEvent represents a staking burn event. +// cborConsensusBurnEvent represents a staking burn event. // See: _oasis-core/go/staking/api/api.go:236-240 -type ConsensusBurnEvent struct { +type cborConsensusBurnEvent struct { Owner [21]byte `json:"owner"` // Oasis address Amount string `json:"amount"` // string representation of quantity.Quantity } -// ConsensusAddEscrowEvent represents an add escrow event. +// cborConsensusAddEscrowEvent represents an add escrow event. // See: _oasis-core/go/staking/api/api.go:266-273 -type ConsensusAddEscrowEvent struct { +type cborConsensusAddEscrowEvent struct { Owner [21]byte `json:"owner"` // Oasis address Escrow [21]byte `json:"escrow"` // Oasis address Amount string `json:"amount"` // string representation of quantity.Quantity NewShares string `json:"new_shares"` // string representation of quantity.Quantity } -// ConsensusReclaimEscrowEvent represents a reclaim escrow event. +// cborConsensusReclaimEscrowEvent represents a reclaim escrow event. // See: _oasis-core/go/staking/api/api.go:313-320 -type ConsensusReclaimEscrowEvent struct { +type cborConsensusReclaimEscrowEvent struct { Owner [21]byte `json:"owner"` // Oasis address Escrow [21]byte `json:"escrow"` // Oasis address Amount string `json:"amount"` // string representation of quantity.Quantity Shares string `json:"shares"` // string representation of quantity.Quantity } + // ============================================================================= -// Tendermint Protobuf Types (minimal definitions to replace tendermint dependency) +// Tendermint Protobuf Deserialization Types // ============================================================================= // Source: github.com/tendermint/tendermint v0.34.21 // These types are used only for protobuf deserialization - no tendermint business logic is needed. diff --git a/badgerdb-sampler/types_evm.go b/badgerdb-sampler/types_evm.go index 5421c52..78cc857 100644 --- a/badgerdb-sampler/types_evm.go +++ b/badgerdb-sampler/types_evm.go @@ -5,12 +5,12 @@ package main type EVMDataInfo struct { StorageType string `json:"storage_type"` // "code", "storage", "block_hash", "confidential_storage" Address string `json:"address,omitempty"` // H160 (20 bytes) - contract address (0x-prefixed hex) - StorageSlot string `json:"storage_slot,omitempty"` // H256 (32 bytes) - storage slot (0x-prefixed hex) + StorageSlot string `json:"storage_slot,omitempty"` // H256 (32 bytes) - storage slot (0x-prefixed hex, truncated) StorageValue string `json:"storage_value,omitempty"` // H256 (32 bytes) - storage value (0x-prefixed hex) RuntimeHeight uint64 `json:"runtime_height,omitempty"` // For block_hash type (runtime block height) - BlockHash string `json:"block_hash,omitempty"` // H256 (32 bytes) - block hash (0x-prefixed hex) - BytecodeSize int `json:"bytecode_size,omitempty"` // Size of contract bytecode - BytecodeRaw string `json:"bytecode_raw,omitempty"` // First 32 bytes of raw bytecode (0x-prefixed hex) + BlockHash string `json:"block_hash,omitempty"` // H256 (32 bytes) - block hash (0x-prefixed hex, truncated) + CodeSize int `json:"code_size,omitempty"` // Size of contract bytecode + CodeHex string `json:"code_hex,omitempty"` // Truncated bytecode (hex, no 0x prefix) } // EVMEventInfo represents a decoded EVM Log event. @@ -23,8 +23,7 @@ type EVMEventInfo struct { EventSignature string `json:"event_signature,omitempty"` // Human-readable signature (if known) EventHash string `json:"event_hash,omitempty"` // topic[0] keccak256 hash (0x-prefixed) DataSize int `json:"data_size"` // Size of non-indexed data - DataRaw string `json:"data_raw,omitempty"` // First 64 bytes of raw data (0x-prefixed hex) - DecodeError string `json:"decode_error,omitempty"` + DataHex string `json:"data_hex,omitempty"` // Truncated raw data (0x-prefixed hex) } // EVMTxInputInfo represents decoded EVM transaction input artifacts. @@ -32,29 +31,29 @@ type EVMEventInfo struct { type EVMTxInputInfo struct { TxHash string `json:"tx_hash"` // Transaction hash (0x-prefixed hex) BatchOrder uint32 `json:"batch_order"` // Order within batch - InputSize int `json:"input_size"` // Raw input size + TxInputSize int `json:"tx_input_size"` // Raw input size + TxInputHex string `json:"tx_input_hex,omitempty"` // Truncated raw input (hex) Method string `json:"method,omitempty"` // SDK method (e.g., "evm.Call") EVMCall *EVMCallInfo `json:"evm_call,omitempty"` // EVM-specific call details - DecodeError string `json:"decode_error,omitempty"` } // EVMCallInfo represents EVM-specific transaction call details. // See: _oasis-sdk/runtime-sdk/modules/evm/src/types.rs:10-16 (Call/Create) type EVMCallInfo struct { - Type string `json:"type"` // "call" or "create" - Address string `json:"address,omitempty"` // H160 target address (0x-prefixed hex) - Value string `json:"value,omitempty"` // U256 value in wei (decimal string) - DataSize int `json:"data_size"` // Size of call data or init code - DataRaw string `json:"data_raw,omitempty"` // First 32 bytes of raw data (0x-prefixed hex) + Type string `json:"type"` // "call" or "create" + Address string `json:"address,omitempty"` // H160 target address (0x-prefixed hex) + Value string `json:"value,omitempty"` // U256 value in wei (decimal string) + DataSize int `json:"data_size"` // Size of call data or init code + DataHex string `json:"data_hex,omitempty"` // Truncated raw data (0x-prefixed hex) } // EVMTxOutputInfo represents decoded EVM transaction output artifacts. // See: _oasis-core/go/runtime/transaction/transaction.go:142-150 (outputArtifacts) type EVMTxOutputInfo struct { - OutputSize int `json:"output_size"` // Raw output size - Success bool `json:"success"` // Transaction succeeded - ResultSize int `json:"result_size,omitempty"` // Size of result data - ResultRaw string `json:"result_raw,omitempty"` // First 32 bytes of raw result (0x-prefixed hex) - Error string `json:"error,omitempty"` // Error message if failed - DecodeError string `json:"decode_error,omitempty"` + TxOutputSize int `json:"tx_output_size"` // Raw output size + TxOutputHex string `json:"tx_output_hex,omitempty"` // Truncated raw output (hex) + Success bool `json:"success"` // Transaction succeeded + ResultSize int `json:"result_size,omitempty"` // Size of result data + ResultHex string `json:"result_hex,omitempty"` // Truncated raw result (0x-prefixed hex) + Error string `json:"error,omitempty"` // Execution error message if failed (kept in child) } diff --git a/badgerdb-sampler/types_runtime.go b/badgerdb-sampler/types_runtime.go index c0d79a2..307497b 100644 --- a/badgerdb-sampler/types_runtime.go +++ b/badgerdb-sampler/types_runtime.go @@ -2,82 +2,6 @@ package main import "github.com/fxamacker/cbor/v2" -// Runtime decode output types - structured representations of decoded runtime database entries. -// These types separate decoding logic from string formatting, enabling flexible output formats. - -// ============================================================================= -// CBOR Deserialization Types (for unmarshaling from database) -// ============================================================================= - -// RuntimeHistoryMetadata represents the metadata stored in runtime history DB. -// See: _oasis-core/go/runtime/history/db.go:34-44 (dbMetadata) -type RuntimeHistoryMetadata struct { - RuntimeID []byte `cbor:"runtime_id"` - Version uint64 `cbor:"version"` - LastConsensusHeight int64 `cbor:"last_consensus_height"` - LastRound uint64 `cbor:"last_round"` -} - -// RuntimeHistoryAnnotatedBlock represents an annotated block in runtime history. -// See: _oasis-core/go/roothash/api/api.go:401-409 (AnnotatedBlock) -type RuntimeHistoryAnnotatedBlock struct { - Height int64 `cbor:"consensus_height"` - Block *RuntimeHistoryBlock `cbor:"block"` -} - -// RuntimeHistoryBlock represents a runtime block. -// See: _oasis-core/go/roothash/api/block/block.go:7-12 (Block) -type RuntimeHistoryBlock struct { - Header RuntimeHistoryBlockHeader `cbor:"header"` -} - -// RuntimeHistoryBlockHeader represents a runtime block header. -// See: _oasis-core/go/roothash/api/block/header.go:69-99 (Header) -type RuntimeHistoryBlockHeader struct { - Version uint16 `cbor:"version"` - Namespace []byte `cbor:"namespace"` - Round uint64 `cbor:"round"` - Timestamp uint64 `cbor:"timestamp"` // POSIX time (Unix seconds) - HeaderType uint8 `cbor:"header_type"` - PreviousHash []byte `cbor:"previous_hash"` - IORoot []byte `cbor:"io_root"` - StateRoot []byte `cbor:"state_root"` - MessagesHash []byte `cbor:"messages_hash"` - InMessagesHash []byte `cbor:"in_msgs_hash"` -} - -// RuntimeHistoryRoundResults represents round results in runtime history. -// See: _oasis-core/go/roothash/api/results.go:5-17 (RoundResults) -type RuntimeHistoryRoundResults struct { - Messages []RuntimeHistoryMessageEvent `cbor:"messages,omitempty"` - GoodComputeEntities [][]byte `cbor:"good_compute_entities,omitempty"` - BadComputeEntities [][]byte `cbor:"bad_compute_entities,omitempty"` -} - -// RuntimeHistoryMessageEvent represents a message event. -// See: _oasis-core/go/roothash/api/api.go:492-499 (MessageEvent) -type RuntimeHistoryMessageEvent struct { - Module string `cbor:"module,omitempty"` - Code uint32 `cbor:"code,omitempty"` - Index uint32 `cbor:"index,omitempty"` - Result cbor.RawMessage `cbor:"result,omitempty"` -} - -// RuntimeInputArtifacts represents input transaction artifacts stored in IO tree. -// See: _oasis-core/go/runtime/transaction/transaction.go:129-140 (inputArtifacts) -type RuntimeInputArtifacts struct { - _ struct{} `cbor:",toarray"` - Input []byte - BatchOrder uint32 -} - -// RuntimeOutputArtifacts represents output transaction artifacts stored in IO tree. -// See: _oasis-core/go/runtime/transaction/transaction.go:145-150 (outputArtifacts) -type RuntimeOutputArtifacts struct { - _ struct{} `cbor:",toarray"` - Output []byte -} - // ============================================================================= // Decoded Output Types (structured representations for JSON output) // ============================================================================= @@ -88,27 +12,31 @@ type RuntimeMkvsKeyInfo struct { KeyType string `json:"key_type"` // "node", "write_log", "roots_metadata", "root_updated_nodes", "metadata", "unknown" RuntimeHeight uint64 `json:"runtime_height,omitempty"` // For write_log, roots_metadata, root_updated_nodes Hash string `json:"hash,omitempty"` // For node (hex, truncated) - DecodeError string `json:"decode_error,omitempty"` } // RuntimeMkvsNodeInfo represents a decoded runtime-mkvs value (node). // See: _oasis-core/go/storage/mkvs/node/node.go:26-32 (prefixes), 294-309 (InternalNode), 531-537 (LeafNode) type RuntimeMkvsNodeInfo struct { - NodeType string `json:"node_type"` // "leaf", "internal", "nil", "non_node", "unknown" - Size int `json:"size"` - Leaf *RuntimeMkvsLeafInfo `json:"leaf,omitempty"` - Internal *RuntimeMkvsInternalInfo `json:"internal,omitempty"` - DecodeError string `json:"decode_error,omitempty"` + NodeType string `json:"node_type"` // "leaf", "internal", "nil", "non_node", "unknown" + NodeSize int `json:"node_size"` // Total MKVS node size (renamed from value_size for clarity) + NodeHex string `json:"node_hex,omitempty"` // Raw node dump (hex, truncated) + Leaf *RuntimeMkvsLeafInfo `json:"leaf,omitempty"` + LeafError string `json:"leaf_error,omitempty"` // Error decoding leaf node + Internal *RuntimeMkvsInternalInfo `json:"internal,omitempty"` + InternalError string `json:"internal_error,omitempty"` // Error decoding internal node + NodeError string `json:"node_error,omitempty"` // Structural parsing error } // RuntimeMkvsLeafInfo represents a decoded MKVS LeafNode. // See: _oasis-core/go/storage/mkvs/node/node.go:531-537 type RuntimeMkvsLeafInfo struct { - Module string `json:"module"` - KeyLen int `json:"key_len"` - Key string `json:"key,omitempty"` // hex, truncated - ValueLen int `json:"value_len"` - DecodedValue *RuntimeLeafValueInfo `json:"decoded_value,omitempty"` + Module string `json:"module"` + KeySize int `json:"key_size"` + KeyHex string `json:"key_hex,omitempty"` // hex, truncated + ValueSize int `json:"value_size"` + ValueHex string `json:"value_hex,omitempty"` // hex, truncated + Value *RuntimeLeafValueInfo `json:"value,omitempty"` // Decoded value (renamed from ValueDecoded) + ValueError string `json:"value_error,omitempty"` // Error decoding value } // RuntimeMkvsInternalInfo represents a decoded MKVS InternalNode. @@ -126,17 +54,16 @@ type RuntimeHistoryKeyInfo struct { KeyType string `json:"key_type"` // "metadata", "block", "round_results", "unknown" RuntimeHeight uint64 `json:"runtime_height,omitempty"` // For block, round_results (runtime block height) ExtraData string `json:"extra,omitempty"` // Unexpected extra bytes (hex) - DecodeError string `json:"decode_error,omitempty"` } // RuntimeHistoryValueInfo represents a decoded runtime-history value. type RuntimeHistoryValueInfo struct { - KeyType string `json:"key_type"` - Size int `json:"size"` - Metadata *RuntimeHistoryMetadataInfo `json:"metadata,omitempty"` - Block *RuntimeHistoryBlockInfo `json:"block,omitempty"` - RoundResults *RuntimeHistoryRoundResultsInfo `json:"round_results,omitempty"` - DecodeError string `json:"decode_error,omitempty"` + Metadata *RuntimeHistoryMetadataInfo `json:"metadata,omitempty"` + MetadataError string `json:"metadata_error,omitempty"` // Error decoding metadata + Block *RuntimeHistoryBlockInfo `json:"block,omitempty"` + BlockError string `json:"block_error,omitempty"` // Error decoding block + RoundResults *RuntimeHistoryRoundResultsInfo `json:"round_results,omitempty"` + RoundResultsError string `json:"round_results_error,omitempty"` // Error decoding round results } // RuntimeHistoryMetadataInfo represents decoded runtime history metadata. @@ -170,15 +97,111 @@ type RuntimeHistoryRoundResultsInfo struct { // RuntimeLeafValueInfo represents a decoded MKVS leaf node value. // See: _oasis-core/go/runtime/transaction/transaction.go:129-150 (artifacts) +// Note: ValueHex and ValueSize are stored in parent RuntimeMkvsLeafInfo to avoid duplication type RuntimeLeafValueInfo struct { - ValueType string `json:"value_type"` // "io_input", "io_output", "io_event", "evm_event", "evm_code", "evm_storage", "evm_block_hash", "cbor", "binary" - InputSize int `json:"input_size,omitempty"` - BatchOrder uint32 `json:"batch_order,omitempty"` - OutputSize int `json:"output_size,omitempty"` - DecodedValue interface{} `json:"decoded_value,omitempty"` // For CBOR/event values - BinarySize int `json:"binary_size,omitempty"` // For binary values - EVM *EVMDataInfo `json:"evm,omitempty"` // For EVM-specific data - EVMEvent *EVMEventInfo `json:"evm_event,omitempty"` // For EVM event data - EVMTxInput *EVMTxInputInfo `json:"evm_tx_input,omitempty"` // For EVM transaction input - EVMTxOutput *EVMTxOutputInfo `json:"evm_tx_output,omitempty"` // For EVM transaction output + ValueType string `json:"value_type,omitempty"` // Computed classification + CBOR interface{} `json:"cbor,omitempty"` // For io_event and cbor types - decoded CBOR data + CBORError string `json:"cbor_error,omitempty"` // Error decoding CBOR + EVM *EVMDataInfo `json:"evm,omitempty"` // For EVM-specific storage data + EVMError string `json:"evm_error,omitempty"` // Error decoding EVM storage + EVMEvent *EVMEventInfo `json:"evm_event,omitempty"` // For EVM event data + EVMEventError string `json:"evm_event_error,omitempty"` // Error decoding EVM event + EVMTxInput *EVMTxInputInfo `json:"evm_tx_input,omitempty"` // For EVM transaction input artifacts + EVMTxInputError string `json:"evm_tx_input_error,omitempty"` // Error decoding EVM tx input + EVMTxOutput *EVMTxOutputInfo `json:"evm_tx_output,omitempty"` // For EVM transaction output artifacts + EVMTxOutputError string `json:"evm_tx_output_error,omitempty"` // Error decoding EVM tx output +} + +// ============================================================================= +// Runtime CBOR Deserialization Types +// ============================================================================= + +// cborRuntimeHistoryMetadata represents the metadata stored in runtime history DB. +// See: _oasis-core/go/runtime/history/db.go:34-44 (dbMetadata) +type cborRuntimeHistoryMetadata struct { + RuntimeID []byte `cbor:"runtime_id"` + Version uint64 `cbor:"version"` + LastConsensusHeight int64 `cbor:"last_consensus_height"` + LastRound uint64 `cbor:"last_round"` +} + +// cborRuntimeHistoryAnnotatedBlock represents an annotated block in runtime history. +// See: _oasis-core/go/roothash/api/api.go:401-409 (AnnotatedBlock) +type cborRuntimeHistoryAnnotatedBlock struct { + Height int64 `cbor:"consensus_height"` + Block *cborRuntimeHistoryBlock `cbor:"block"` +} + +// cborRuntimeHistoryBlock represents a runtime block. +// See: _oasis-core/go/roothash/api/block/block.go:7-12 (Block) +type cborRuntimeHistoryBlock struct { + Header cborRuntimeHistoryBlockHeader `cbor:"header"` +} + +// cborRuntimeHistoryBlockHeader represents a runtime block header. +// See: _oasis-core/go/roothash/api/block/header.go:69-99 (Header) +type cborRuntimeHistoryBlockHeader struct { + Version uint16 `cbor:"version"` + Namespace []byte `cbor:"namespace"` + Round uint64 `cbor:"round"` + Timestamp uint64 `cbor:"timestamp"` // POSIX time (Unix seconds) + HeaderType uint8 `cbor:"header_type"` + PreviousHash []byte `cbor:"previous_hash"` + IORoot []byte `cbor:"io_root"` + StateRoot []byte `cbor:"state_root"` + MessagesHash []byte `cbor:"messages_hash"` + InMessagesHash []byte `cbor:"in_msgs_hash"` +} + +// cborRuntimeHistoryRoundResults represents round results in runtime history. +// See: _oasis-core/go/roothash/api/results.go:5-17 (RoundResults) +type cborRuntimeHistoryRoundResults struct { + Messages []cborRuntimeHistoryMessageEvent `cbor:"messages,omitempty"` + GoodComputeEntities [][]byte `cbor:"good_compute_entities,omitempty"` + BadComputeEntities [][]byte `cbor:"bad_compute_entities,omitempty"` +} + +// cborRuntimeHistoryMessageEvent represents a message event. +// See: _oasis-core/go/roothash/api/api.go:492-499 (MessageEvent) +type cborRuntimeHistoryMessageEvent struct { + Module string `cbor:"module,omitempty"` + Code uint32 `cbor:"code,omitempty"` + Index uint32 `cbor:"index,omitempty"` + Result cbor.RawMessage `cbor:"result,omitempty"` +} + +// cborRuntimeInputArtifacts represents input transaction artifacts stored in IO tree. +// See: _oasis-core/go/runtime/transaction/transaction.go:129-140 (inputArtifacts) +type cborRuntimeInputArtifacts struct { + _ struct{} `cbor:",toarray"` + Input []byte + BatchOrder uint32 +} + +// cborRuntimeOutputArtifacts represents output transaction artifacts stored in IO tree. +// See: _oasis-core/go/runtime/transaction/transaction.go:145-150 (outputArtifacts) +type cborRuntimeOutputArtifacts struct { + _ struct{} `cbor:",toarray"` + Output []byte +} + +// cborRuntimeCallArrayFormat represents runtime Call structure encoded as CBOR array (old format). +// See: _oasis-sdk/runtime-sdk/src/types/transaction.rs:124-141 (Call) +// Used in older runtime versions before map-based encoding +type cborRuntimeCallArrayFormat struct { + _ struct{} `cbor:",toarray"` + Format uint8 // CallFormat: 0=Plain, 1=EncryptedX25519DeoxysII + Method string // Method name + Body cbor.RawMessage // Method body (CBOR-encoded) + ReadOnly bool // Read-only flag +} + +// cborRuntimeCallMapFormat represents runtime Call structure encoded as CBOR map (new format). +// See: _oasis-sdk/runtime-sdk/src/types/transaction.rs:124-141 (Call) +// Used in newer runtime versions +type cborRuntimeCallMapFormat struct { + Format uint8 `cbor:"format,omitempty"` + Method string `cbor:"method,omitempty"` + Body cbor.RawMessage `cbor:"body"` + ReadOnly bool `cbor:"ro,omitempty"` } diff --git a/badgerdb-sampler/utils.go b/badgerdb-sampler/utils.go index 6285306..ef61949 100644 --- a/badgerdb-sampler/utils.go +++ b/badgerdb-sampler/utils.go @@ -8,6 +8,13 @@ import ( "github.com/fxamacker/cbor/v2" ) +const ( + // TruncateHashSize is the default hex character limit for hash truncation + TruncateHashSize = 16 + // TruncateLongSize is the default hex character limit for long data truncation + TruncateLongSize = 500 +) + // extractModuleName extracts module name from MKVS leaf key and returns module:subtype description func extractModuleName(key []byte) string { if len(key) == 0 { @@ -31,9 +38,9 @@ func extractModuleName(key []byte) string { } else if kind == 2 { kindStr = "output" } - return fmt.Sprintf("io_tx:%s (hash=%s)", kindStr, truncateHex(key[1:33], 16)) + return fmt.Sprintf("io_tx:%s (hash=%s)", kindStr, truncateHex(key[1:33], TruncateHashSize)) } - return fmt.Sprintf("io_tx (hash=%s)", truncateHex(key[1:], 16)) + return fmt.Sprintf("io_tx (hash=%s)", truncateHex(key[1:], TruncateHashSize)) case 'E': // Event tag prefix (0x45) // Key format: 'E' + tag_key (variable, module name) + tx_hash (32 bytes) @@ -70,7 +77,7 @@ func extractModuleName(key []byte) string { subKey := key[end:] return describeModuleKey(moduleName, subKey) } - return truncateHex(key, 16) + return truncateHex(key, TruncateHashSize) } // describeModuleKey returns module name with sub-key type description @@ -182,6 +189,43 @@ func formatCBOR(v interface{}, rawLen int) string { } } +// formatCBORDetailed formats decoded CBOR value with detailed structure +func formatCBORDetailed(v interface{}) interface{} { + switch val := v.(type) { + case map[interface{}]interface{}: + result := make(map[string]interface{}) + for k, v := range val { + keyStr := fmt.Sprintf("%v", k) + result[keyStr] = formatCBORDetailed(v) + } + return result + case []interface{}: + result := make([]interface{}, len(val)) + for i, elem := range val { + result[i] = formatCBORDetailed(elem) + } + return result + case []byte: + // Format as hex string with size info + if len(val) <= 32 { + return truncateHex0x(val, 64) + } + return truncateHex0x(val, 64) + fmt.Sprintf(" (%d bytes)", len(val)) + case string: + return val + case uint64: + return val + case int64: + return val + case bool: + return val + case nil: + return nil + default: + return fmt.Sprintf("%v", val) + } +} + // isPrintableASCII checks if a string contains only printable ASCII characters func isPrintableASCII(s string) bool { if len(s) == 0 { From 38eecf218e708b00a3e8c2707e32b42eefa1791a Mon Sep 17 00:00:00 2001 From: gw0 Date: Thu, 27 Nov 2025 21:51:08 +0100 Subject: [PATCH 15/22] badgerdb-sampler: Add decoding of consensus events and improve de/serialization --- badgerdb-sampler/decode_consensus.go | 184 ++++++++++++++++++++++----- badgerdb-sampler/types_consensus.go | 112 +++++++++++----- badgerdb-sampler/utils.go | 28 ---- 3 files changed, 231 insertions(+), 93 deletions(-) diff --git a/badgerdb-sampler/decode_consensus.go b/badgerdb-sampler/decode_consensus.go index 539b7e1..be693bf 100644 --- a/badgerdb-sampler/decode_consensus.go +++ b/badgerdb-sampler/decode_consensus.go @@ -1,6 +1,7 @@ package main import ( + "encoding/base64" "encoding/binary" "fmt" "strconv" @@ -380,14 +381,14 @@ func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueIn if keySize == 22 { switch key[0] { case 0x34, 0x35, 0x36, 0x37: // staking accounts, delegations, debonding, allowances - leaf.OasisAddress = bech32Encode("oasis", key[1:22]) + leaf.OasisAddress = (*(*OasisAddress)(key[1:22])).String() } } // Entity/node keys also contain addresses if keySize >= 22 { switch key[0] { case 0x40, 0x41: // registry entities, nodes - leaf.OasisAddress = bech32Encode("oasis", key[1:22]) + leaf.OasisAddress = (*(*OasisAddress)(key[1:22])).String() } } @@ -755,18 +756,26 @@ func decodeConsensusStateValue(keyType string, value []byte) ConsensusStateValue if resp.DeliverTxs != nil { for _, txResult := range resp.DeliverTxs { - // Count events + // Count and decode events for _, event := range txResult.Events { eventSummary.EventTypeCounts[event.Type]++ + + // Decode event if sample limit not reached + if decodedEvents, err := decodeConsensusEvent(event); err == nil { + for _, decodedEvent := range decodedEvents { + if len(eventSummary.Events) < 10 { + eventSummary.Events = append(eventSummary.Events, decodedEvent) + } + } + } } // Decode transaction if present if len(txResult.Data) > 0 { - decodedTx, err := decodeCBORTransaction(txResult.Data) - if err == "" { + if decodedTx, err := decodeConsensusTransaction(txResult.Data); err == nil { txSummary.MethodCounts[decodedTx.Method]++ // Only include first few transactions for sample - if len(txSummary.Transactions) < 5 { + if len(txSummary.Transactions) < 10 { txSummary.Transactions = append(txSummary.Transactions, decodedTx) } } @@ -774,14 +783,19 @@ func decodeConsensusStateValue(keyType string, value []byte) ConsensusStateValue } } - if len(txSummary.MethodCounts) > 0 { - abciResponseInfo.TransactionSummary = txSummary - } - // Events from BeginBlock if resp.BeginBlock != nil { for _, event := range resp.BeginBlock.Events { eventSummary.EventTypeCounts[event.Type]++ + + // Decode event if sample limit not reached + if decodedEvents, err := decodeConsensusEvent(event); err == nil { + for _, decodedEvent := range decodedEvents { + if len(eventSummary.Events) < 10 { + eventSummary.Events = append(eventSummary.Events, decodedEvent) + } + } + } } } @@ -789,9 +803,23 @@ func decodeConsensusStateValue(keyType string, value []byte) ConsensusStateValue if resp.EndBlock != nil { for _, event := range resp.EndBlock.Events { eventSummary.EventTypeCounts[event.Type]++ + + // Decode event if sample limit not reached + if decodedEvents, err := decodeConsensusEvent(event); err == nil { + for _, decodedEvent := range decodedEvents { + if len(eventSummary.Events) < 10 { + eventSummary.Events = append(eventSummary.Events, decodedEvent) + } + } + } } } + + if len(txSummary.MethodCounts) > 0 { + abciResponseInfo.TransactionSummary = txSummary + } + if len(eventSummary.EventTypeCounts) > 0 { abciResponseInfo.EventSummary = eventSummary } @@ -851,33 +879,33 @@ func decodeConsensusStateValue(keyType string, value []byte) ConsensusStateValue return info } -// decodeCBORTransaction decodes a raw CBOR-encoded transaction bytes into ConsensusTransactionInfo. +// decodeConsensusTransaction decodes a raw CBOR-encoded transaction bytes into ConsensusTransactionInfo. // Returns (decoded, error) where error is a parsing error. // See: _oasis-core/go/consensus/api/transaction/transaction.go:42-54 // See: _oasis-core/go/common/crypto/signature/signature.go:415-421 // See: _oasis-core/go/staking/api/api.go for transaction body types -func decodeCBORTransaction(rawTx []byte) (ConsensusTransactionInfo, string) { - decoded := ConsensusTransactionInfo{} +func decodeConsensusTransaction(rawTx []byte) (ConsensusTransactionInfo, error) { + info := ConsensusTransactionInfo{} // Decode SignedTransaction envelope var signedTx cborConsensusSignedTransaction if err := cbor.Unmarshal(rawTx, &signedTx); err != nil { - return decoded, fmt.Sprintf("failed to decode SignedTransaction: %v", err) + return info, fmt.Errorf("failed to decode SignedTransaction: %w", err) } // Extract signer public key - decoded.Signer = truncateHex(signedTx.Signature.PublicKey[:], TruncateLongSize) + info.Signer = truncateHex(signedTx.Signature.PublicKey[:], TruncateLongSize) // Decode inner Transaction from the blob var tx cborConsensusInnerTransaction if err := cbor.Unmarshal(signedTx.Blob, &tx); err != nil { - return decoded, fmt.Sprintf("failed to decode Transaction: %v", err) + return info, fmt.Errorf("failed to decode Transaction: %w", err) } - decoded.Nonce = tx.Nonce - decoded.Method = tx.Method + info.Nonce = tx.Nonce + info.Method = tx.Method if tx.Fee != nil { - decoded.Fee = &ConsensusFee{ + info.Fee = &ConsensusFee{ Amount: tx.Fee.Amount, Gas: tx.Fee.Gas, } @@ -885,69 +913,155 @@ func decodeCBORTransaction(rawTx []byte) (ConsensusTransactionInfo, string) { // Decode body based on method if len(tx.Body) > 0 { - decoded.BodyPreview = truncateHex(tx.Body, TruncateLongSize) + info.BodyHex = truncateHex(tx.Body, TruncateLongSize) + info.BodySize = len(tx.Body) switch tx.Method { case "staking.Transfer": var transfer cborConsensusTransfer if err := cbor.Unmarshal(tx.Body, &transfer); err == nil { - transfer.Amount = quantityBytesToString(tx.Body, "amount") - decoded.DecodedBody = transfer + info.Body = transfer + } else { + info.BodyError = fmt.Sprintf("failed to decode transfer body: %v", err) } case "staking.Burn": var burn cborConsensusBurn if err := cbor.Unmarshal(tx.Body, &burn); err == nil { - burn.Amount = quantityBytesToString(tx.Body, "amount") - decoded.DecodedBody = burn + info.Body = burn + } else { + info.BodyError = fmt.Sprintf("failed to decode burn body: %v", err) } case "staking.AddEscrow": var escrow cborConsensusAddEscrow if err := cbor.Unmarshal(tx.Body, &escrow); err == nil { - escrow.Amount = quantityBytesToString(tx.Body, "amount") - decoded.DecodedBody = escrow + info.Body = escrow + } else { + info.BodyError = fmt.Sprintf("failed to decode add_escrow body: %v", err) } case "staking.ReclaimEscrow": var reclaim cborConsensusReclaimEscrow if err := cbor.Unmarshal(tx.Body, &reclaim); err == nil { - reclaim.Shares = quantityBytesToString(tx.Body, "shares") - decoded.DecodedBody = reclaim + info.Body = reclaim + } else { + info.BodyError = fmt.Sprintf("failed to decode reclaim_escrow body: %v", err) } case "registry.RegisterEntity": var regEntity cborConsensusRegisterEntity if err := cbor.Unmarshal(tx.Body, ®Entity); err == nil { - decoded.DecodedBody = regEntity + info.Body = regEntity + } else { + info.BodyError = fmt.Sprintf("failed to decode register_entity body: %v", err) } case "registry.RegisterNode": var regNode cborConsensusRegisterNode if err := cbor.Unmarshal(tx.Body, ®Node); err == nil { - decoded.DecodedBody = regNode + info.Body = regNode + } else { + info.BodyError = fmt.Sprintf("failed to decode register_node body: %v", err) } case "roothash.ExecutorCommit": var execCommit cborConsensusExecutorCommit if err := cbor.Unmarshal(tx.Body, &execCommit); err == nil { - decoded.DecodedBody = execCommit + info.Body = execCommit + } else { + info.BodyError = fmt.Sprintf("failed to decode executor_commit body: %v", err) } case "governance.SubmitProposal": var proposal cborConsensusSubmitProposal if err := cbor.Unmarshal(tx.Body, &proposal); err == nil { - proposal.Deposit = quantityBytesToString(tx.Body, "deposit") - decoded.DecodedBody = proposal + info.Body = proposal + } else { + info.BodyError = fmt.Sprintf("failed to decode submit_proposal body: %v", err) } case "governance.CastVote": var vote cborConsensusCastVote if err := cbor.Unmarshal(tx.Body, &vote); err == nil { - decoded.DecodedBody = vote + info.Body = vote + } else { + info.BodyError = fmt.Sprintf("failed to decode cast_vote body: %v", err) } } } - return decoded, "" + return info, nil +} + +// decodeConsensusEvent decodes a Tendermint event into ConsensusEventInfo. +// Events use base64-encoded CBOR-marshaled bodies in the Value field. +// Returns all decoded attributes. On success returns (results, nil). +// See: _oasis-core/go/consensus/api/events/events.go (TypedAttribute pattern) +// See: _oasis-core/go/staking/api/api.go (event type definitions) +func decodeConsensusEvent(event tmEvent) ([]ConsensusEventInfo, error) { + if len(event.Attributes) == 0 { + return nil, fmt.Errorf("event has no attributes") + } + + var results []ConsensusEventInfo + + // Process all attributes + for _, attr := range event.Attributes { + info := ConsensusEventInfo{ + EventType: event.Type, + EventKind: string(attr.Key), + } + + // Base64 decode the attribute value + cborData, err := base64.StdEncoding.DecodeString(string(attr.Value)) + if err != nil { + info.BodyError = fmt.Sprintf("base64 decode failed: %v", err) + results = append(results, info) + continue + } + + // Populate raw body fields + info.BodyHex = truncateHex(cborData, TruncateLongSize) + info.BodySize = len(cborData) + + // Decode body based on event kind + switch info.EventKind { + case "transfer": + var transfer cborConsensusTransferEvent + if err := cbor.Unmarshal(cborData, &transfer); err == nil { + info.Body = transfer + } else { + info.BodyError = fmt.Sprintf("cbor unmarshal failed: %v", err) + } + + case "burn": + var burn cborConsensusBurnEvent + if err := cbor.Unmarshal(cborData, &burn); err == nil { + info.Body = burn + } else { + info.BodyError = fmt.Sprintf("cbor unmarshal failed: %v", err) + } + + case "add_escrow": + var addEscrow cborConsensusAddEscrowEvent + if err := cbor.Unmarshal(cborData, &addEscrow); err == nil { + info.Body = addEscrow + } else { + info.BodyError = fmt.Sprintf("cbor unmarshal failed: %v", err) + } + + case "reclaim_escrow": + var reclaimEscrow cborConsensusReclaimEscrowEvent + if err := cbor.Unmarshal(cborData, &reclaimEscrow); err == nil { + info.Body = reclaimEscrow + } else { + info.BodyError = fmt.Sprintf("cbor unmarshal failed: %v", err) + } + } + + results = append(results, info) + } + + return results, nil } diff --git a/badgerdb-sampler/types_consensus.go b/badgerdb-sampler/types_consensus.go index 6fe8cc8..5f00803 100644 --- a/badgerdb-sampler/types_consensus.go +++ b/badgerdb-sampler/types_consensus.go @@ -1,6 +1,8 @@ package main import ( + "encoding/json" + "math/big" "time" "github.com/fxamacker/cbor/v2" @@ -197,13 +199,15 @@ type ConsensusTransactionSummary struct { // ConsensusTransactionInfo represents a decoded Oasis consensus transaction. // See: _oasis-core/go/consensus/api/transaction/transaction.go:42-54 type ConsensusTransactionInfo struct { - Nonce uint64 `json:"nonce"` - Method string `json:"method"` - Fee *ConsensusFee `json:"fee,omitempty"` - Signer string `json:"signer,omitempty"` // hex-encoded public key - TxHash string `json:"tx_hash,omitempty"` // hex-encoded transaction hash - BodyPreview string `json:"body_preview,omitempty"` // hex preview if body not decoded - DecodedBody interface{} `json:"decoded_body,omitempty"` // Decoded body for known types + Nonce uint64 `json:"nonce"` + Method string `json:"method"` + Fee *ConsensusFee `json:"fee,omitempty"` + Signer string `json:"signer,omitempty"` // hex-encoded public key + TxHash string `json:"tx_hash,omitempty"` // hex-encoded transaction hash + BodyHex string `json:"body_hex,omitempty"` // Truncated hex dump of raw body + BodySize int `json:"body_size,omitempty"` // Size of body in bytes + Body interface{} `json:"body,omitempty"` // Decoded body for known types + BodyError string `json:"body_error,omitempty"` // Error decoding body } // ConsensusFee represents transaction fee. @@ -215,7 +219,18 @@ type ConsensusFee struct { // ConsensusEventSummary summarizes decoded events from ABCI responses. type ConsensusEventSummary struct { - EventTypeCounts map[string]int `json:"event_type_counts"` // Count by event type + EventTypeCounts map[string]int `json:"event_type_counts"` // Count by event type + Events []ConsensusEventInfo `json:"events,omitempty"` // Sample of decoded events +} + +// ConsensusEventInfo represents a decoded consensus event (unified type for all event kinds). +type ConsensusEventInfo struct { + EventType string `json:"event_type"` // "staking.transfer", "staking.burn", etc. + EventKind string `json:"event_kind"` // "transfer", "burn", "add_escrow", "reclaim_escrow" + BodyHex string `json:"body_hex,omitempty"` // Truncated hex dump of raw body + BodySize int `json:"body_size,omitempty"` // Size of body in bytes + Body interface{} `json:"body,omitempty"` // Decoded body for known types + BodyError string `json:"body_error,omitempty"` // Error decoding body } @@ -223,6 +238,43 @@ type ConsensusEventSummary struct { // Consensus CBOR Deserialization Types // ============================================================================= +// QuantityBytes wraps big.Int and implements encoding.BinaryUnmarshaler to automatically +// decode CBOR byte strings (big-endian) into arbitrary-precision unsigned integers. +// See: _oasis-core/go/common/quantity/quantity.go:28-50 (MarshalBinary/UnmarshalBinary) +type QuantityBytes big.Int + +// UnmarshalBinary implements encoding.BinaryUnmarshaler. +// Decodes a byte slice (big-endian) into a big.Int. +func (q *QuantityBytes) UnmarshalBinary(data []byte) error { + if q == nil { + return nil + } + (*big.Int)(q).SetBytes(data) + return nil +} + +// String returns the decimal string representation of the quantity. +func (q *QuantityBytes) String() string { + if q == nil { + return "0" + } + return (*big.Int)(q).String() +} + +// OasisAddress represents a 21-byte Oasis address with automatic bech32 encoding. +// See: _oasis-core/go/common/crypto/address/address.go (ADDRESS_SIZE = 21) +type OasisAddress [21]byte + +// MarshalJSON implements json.Marshaler to automatically encode as bech32 string. +func (a OasisAddress) MarshalJSON() ([]byte, error) { + return json.Marshal(bech32Encode("oasis", a[:])) +} + +// String returns the bech32-encoded address (e.g., "oasis1..."). +func (a OasisAddress) String() string { + return bech32Encode("oasis", a[:]) +} + // cborConsensusSignedTransaction represents the signature envelope. // See: _oasis-core/go/common/crypto/signature/signature.go:415-421 type cborConsensusSignedTransaction struct { @@ -256,28 +308,28 @@ type cborConsensusSignature struct { // cborConsensusTransfer represents a staking transfer transaction body. // See: _oasis-core/go/staking/api/api.go:342-345 type cborConsensusTransfer struct { - To [21]byte `json:"to"` // Oasis address - Amount string `json:"amount"` // string representation of quantity.Quantity + To OasisAddress `json:"to"` // Oasis address (bech32-encoded in JSON) + Amount *QuantityBytes `json:"amount"` // QuantityBytes automatically unmarshals from CBOR byte string } // cborConsensusBurn represents a staking burn transaction body. // See: _oasis-core/go/staking/api/api.go:368-370 type cborConsensusBurn struct { - Amount string `json:"amount"` // string representation of quantity.Quantity + Amount *QuantityBytes `json:"amount"` // QuantityBytes automatically unmarshals from CBOR byte string } // cborConsensusAddEscrow represents an add escrow transaction body. // See: _oasis-core/go/staking/api/api.go:403-406 type cborConsensusAddEscrow struct { - Account [21]byte `json:"account"` // Oasis address - Amount string `json:"amount"` // string representation of quantity.Quantity + Account OasisAddress `json:"account"` // Oasis address (bech32-encoded in JSON) + Amount *QuantityBytes `json:"amount"` // QuantityBytes automatically unmarshals from CBOR byte string } // cborConsensusReclaimEscrow represents a reclaim escrow transaction body. // See: _oasis-core/go/staking/api/api.go:444-447 type cborConsensusReclaimEscrow struct { - Account [21]byte `json:"account"` // Oasis address - Shares string `json:"shares"` // string representation of quantity.Quantity + Account OasisAddress `json:"account"` // Oasis address (bech32-encoded in JSON) + Shares *QuantityBytes `json:"shares"` // QuantityBytes automatically unmarshals from CBOR byte string } // cborConsensusRegisterEntity represents a registry entity registration transaction body. @@ -304,8 +356,8 @@ type cborConsensusExecutorCommit struct { // cborConsensusSubmitProposal represents a governance proposal submission transaction body. // See: _oasis-core/go/governance/api/api.go type cborConsensusSubmitProposal struct { - Content []byte `json:"content,omitempty"` // CBOR-encoded proposal content - Deposit string `json:"deposit,omitempty"` // string representation of quantity.Quantity + Content []byte `json:"content,omitempty"` // CBOR-encoded proposal content + Deposit *QuantityBytes `json:"deposit,omitempty"` // QuantityBytes automatically unmarshals from CBOR byte string } // cborConsensusCastVote represents a governance vote transaction body. @@ -318,34 +370,34 @@ type cborConsensusCastVote struct { // cborConsensusTransferEvent represents a staking transfer event. // See: _oasis-core/go/staking/api/api.go:223-229 type cborConsensusTransferEvent struct { - From [21]byte `json:"from"` // Oasis address - To [21]byte `json:"to"` // Oasis address - Amount string `json:"amount"` // string representation of quantity.Quantity + From OasisAddress `json:"from"` // Oasis address (bech32-encoded in JSON) + To OasisAddress `json:"to"` // Oasis address (bech32-encoded in JSON) + Amount *QuantityBytes `json:"amount"` // QuantityBytes automatically unmarshals from CBOR byte string } // cborConsensusBurnEvent represents a staking burn event. // See: _oasis-core/go/staking/api/api.go:236-240 type cborConsensusBurnEvent struct { - Owner [21]byte `json:"owner"` // Oasis address - Amount string `json:"amount"` // string representation of quantity.Quantity + Owner OasisAddress `json:"owner"` // Oasis address (bech32-encoded in JSON) + Amount *QuantityBytes `json:"amount"` // QuantityBytes automatically unmarshals from CBOR byte string } // cborConsensusAddEscrowEvent represents an add escrow event. // See: _oasis-core/go/staking/api/api.go:266-273 type cborConsensusAddEscrowEvent struct { - Owner [21]byte `json:"owner"` // Oasis address - Escrow [21]byte `json:"escrow"` // Oasis address - Amount string `json:"amount"` // string representation of quantity.Quantity - NewShares string `json:"new_shares"` // string representation of quantity.Quantity + Owner OasisAddress `json:"owner"` // Oasis address (bech32-encoded in JSON) + Escrow OasisAddress `json:"escrow"` // Oasis address (bech32-encoded in JSON) + Amount *QuantityBytes `json:"amount"` // QuantityBytes automatically unmarshals from CBOR byte string + NewShares *QuantityBytes `json:"new_shares"` // QuantityBytes automatically unmarshals from CBOR byte string } // cborConsensusReclaimEscrowEvent represents a reclaim escrow event. // See: _oasis-core/go/staking/api/api.go:313-320 type cborConsensusReclaimEscrowEvent struct { - Owner [21]byte `json:"owner"` // Oasis address - Escrow [21]byte `json:"escrow"` // Oasis address - Amount string `json:"amount"` // string representation of quantity.Quantity - Shares string `json:"shares"` // string representation of quantity.Quantity + Owner OasisAddress `json:"owner"` // Oasis address (bech32-encoded in JSON) + Escrow OasisAddress `json:"escrow"` // Oasis address (bech32-encoded in JSON) + Amount *QuantityBytes `json:"amount"` // QuantityBytes automatically unmarshals from CBOR byte string + Shares *QuantityBytes `json:"shares"` // QuantityBytes automatically unmarshals from CBOR byte string } diff --git a/badgerdb-sampler/utils.go b/badgerdb-sampler/utils.go index ef61949..68b49f3 100644 --- a/badgerdb-sampler/utils.go +++ b/badgerdb-sampler/utils.go @@ -4,8 +4,6 @@ import ( "fmt" "math/big" "strings" - - "github.com/fxamacker/cbor/v2" ) const ( @@ -345,29 +343,3 @@ func bech32CreateChecksum(hrp string, data []int) []int { return checksum } -// quantityBytesToString extracts a quantity.Quantity field from CBOR bytes and converts to string. -// This is a simplified parser that decodes the CBOR to get the nested big.Int value. -// See: _oasis-core/go/common/quantity/quantity.go:28-30 -func quantityBytesToString(cborBytes []byte, fieldName string) string { - // Decode to a generic map to extract the field - var data map[string]interface{} - if err := cbor.Unmarshal(cborBytes, &data); err != nil { - return "" - } - - // Get the field value - if val, ok := data[fieldName]; ok { - // quantity.Quantity is encoded as a byte slice containing the big.Int bytes - if qtyBytes, ok := val.([]byte); ok { - // Convert bytes to big.Int - var bi big.Int - bi.SetBytes(qtyBytes) - return bi.String() - } - // Try direct conversion - return fmt.Sprintf("%v", val) - } - - return "" -} - From f4596465ba8682bcfc867783ae2340267d14a993 Mon Sep 17 00:00:00 2001 From: gw0 Date: Thu, 27 Nov 2025 23:51:01 +0100 Subject: [PATCH 16/22] badgerdb-sampler: Improve database opening --- badgerdb-sampler/db_common.go | 47 +++++++++++++++++ badgerdb-sampler/db_v2.go | 96 +++++++++++++++++++++++++++++---- badgerdb-sampler/db_v3.go | 99 ++++++++++++++++++++++++++++++----- badgerdb-sampler/db_v4.go | 99 ++++++++++++++++++++++++++++++----- badgerdb-sampler/main.go | 4 +- badgerdb-sampler/utils.go | 2 +- 6 files changed, 309 insertions(+), 38 deletions(-) diff --git a/badgerdb-sampler/db_common.go b/badgerdb-sampler/db_common.go index acc3a1d..97b1805 100644 --- a/badgerdb-sampler/db_common.go +++ b/badgerdb-sampler/db_common.go @@ -95,3 +95,50 @@ func copyFile(src, dst string) error { return dstFile.Sync() } + +// removeMemtableFiles renames .mem files by appending .bak suffix and returns a restore function. +// This allows opening databases with corrupted memtables by letting BadgerDB create fresh memtable files. +func removeMemtableFiles(dbPath string) (backupDir string, restore func() error, err error) { + entries, err := os.ReadDir(dbPath) + if err != nil { + return "", nil, fmt.Errorf("failed to read database directory: %w", err) + } + + // Rename all .mem files to .mem.bak + count := 0 + for _, entry := range entries { + if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".mem") { + oldPath := filepath.Join(dbPath, entry.Name()) + newPath := oldPath + ".bak" + if err := os.Rename(oldPath, newPath); err != nil { + return "", nil, fmt.Errorf("failed to rename %s: %w", entry.Name(), err) + } + count++ + } + } + + fmt.Fprintf(os.Stderr, " Renamed %d memtable file(s) to *.mem.bak\n", count) + + // Restore function renames .mem.bak files back to .mem + restore = func() error { + entries, err := os.ReadDir(dbPath) + if err != nil { + return fmt.Errorf("failed to read database directory: %w", err) + } + count := 0 + for _, entry := range entries { + if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".mem.bak") { + oldPath := filepath.Join(dbPath, entry.Name()) + newPath := strings.TrimSuffix(oldPath, ".bak") + if err := os.Rename(oldPath, newPath); err != nil { + return fmt.Errorf("failed to restore %s: %w", entry.Name(), err) + } + count++ + } + } + fmt.Fprintf(os.Stderr, " Restored %d memtable file(s) from *.mem.bak\n", count) + return nil + } + + return dbPath, restore, nil +} diff --git a/badgerdb-sampler/db_v2.go b/badgerdb-sampler/db_v2.go index a0c21ad..728d6db 100644 --- a/badgerdb-sampler/db_v2.go +++ b/badgerdb-sampler/db_v2.go @@ -24,18 +24,20 @@ func init() { BadgerVersion = "v2.2007.2" } -// openDatabase tries to open the database in ReadOnly mode first, -// then falls back to local mirror mode for corrupted/improperly-closed databases on FUSE +// openDatabase tries three strategies in order: +// 1. ReadOnly mode (cleanly closed databases) +// 2. Minimal read-write mode (allows log replay on original path) +// 3. Local mirror mode (FUSE workaround with mmap-safe copy) func openDatabase(path string) (*DB, error) { - // Try ReadOnly mode first (clean databases) + // Stage 1: Try read-only mode first (clean databases) + fmt.Fprintf(os.Stderr, "Opening in read-only mode...\n") opts := badger.DefaultOptions(path) opts.ReadOnly = true // Keep default logger enabled for diagnostics db, err := badger.Open(opts) if err == nil { - // Successfully opened - fmt.Fprintf(os.Stderr, "Opened database in ReadOnly mode\n") + fmt.Fprintf(os.Stderr, "Successfully opened in read-only mode\n") return db, nil } @@ -44,7 +46,79 @@ func openDatabase(path string) (*DB, error) { return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) } - fmt.Fprintf(os.Stderr, "Failed, trying local mirror workaround for FUSE...\n") + fmt.Fprintf(os.Stderr, "Failed to open in read-only mode: %v\n", err) + + // Stage 2: Try minimal read-write mode (allows log replay) + fmt.Fprintf(os.Stderr, "Opening in minimal read-write mode (allows log replay)...\n") + opts = badger.DefaultOptions(path) + opts.ReadOnly = false + opts.SyncWrites = false + opts.NumMemtables = 1 // Minimize memtable usage + opts.MaxTableSize = 1 << 20 // 1MB table size (minimal) + opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) + opts.NumLevelZeroTables = 100 + opts.NumLevelZeroTablesStall = 200 + opts.NumCompactors = 0 + opts.CompactL0OnClose = false + opts.BypassLockGuard = true // Allow concurrent access + opts.DetectConflicts = false // Reduce overhead + // Keep default logger enabled for diagnostics + + db, err = badger.Open(opts) + if err == nil { + fmt.Fprintf(os.Stderr, "Successfully opened in minimal read-write mode\n") + return db, nil + } + + // Check again for version incompatibility + if strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) + } + + fmt.Fprintf(os.Stderr, "Failed to open in minimal read-write mode: %v\n", err) + + // Stage 3: Try removing corrupted memtable files and retry + fmt.Fprintf(os.Stderr, "Opening without memtable files...\n") + if backupDir, restoreMemFiles, err := removeMemtableFiles(path); backupDir != "" { + // Memtable files were removed, retry opening + opts = badger.DefaultOptions(path) + opts.ReadOnly = false + opts.SyncWrites = false + opts.NumMemtables = 1 // Minimize memtable usage + opts.MaxTableSize = 1 << 20 // 1MB memtable (minimal) - v2 uses MaxTableSize instead of MemTableSize + opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) + opts.NumLevelZeroTables = 100 + opts.NumLevelZeroTablesStall = 200 + opts.NumCompactors = 0 + opts.CompactL0OnClose = false + opts.BypassLockGuard = true // Allow concurrent access + opts.DetectConflicts = false // Reduce overhead + // Keep default logger enabled for diagnostics + + db, err = badger.Open(opts) + if err == nil { + fmt.Fprintf(os.Stderr, "Successfully opened without memtable files\n") + return db, nil + } + + // Check for version incompatibility + if strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) + } + + // Restore memtable files before continuing + fmt.Fprintf(os.Stderr, "Restoring memtable files...\n") + if restoreErr := restoreMemFiles(); restoreErr != nil { + fmt.Fprintf(os.Stderr, "Warning: Failed to restore memtable files: %v\n", restoreErr) + } else { + fmt.Fprintf(os.Stderr, "Memtable files restored\n") + } + } + + fmt.Fprintf(os.Stderr, "Failed to open without memtable files: %v\n", err) + + // Stage 4: Try local mirror workaround (FUSE compatibility) + fmt.Fprintf(os.Stderr, "Opening with local mirror (workaround for SIGBUS error on FUSE)...\n") // Create local mirror with symlinks to avoid mmap SIGBUS errors on FUSE tmpDir, cleanup, err := createLocalMirror(path) @@ -55,8 +129,7 @@ func openDatabase(path string) (*DB, error) { // This is acceptable for read-only analysis tools. For long-running services, use defer cleanup() _ = cleanup - // Open from local mirror in read-write mode (needed for corrupted/improperly closed DBs) - // Unified configuration across all BadgerDB versions + // Open from local mirror with same minimal read-write settings opts = badger.DefaultOptions(tmpDir) opts.ReadOnly = false opts.SyncWrites = false @@ -73,8 +146,7 @@ func openDatabase(path string) (*DB, error) { db, err = badger.Open(opts) if err == nil { - // Successfully opened - fmt.Fprintf(os.Stderr, "Opened database from local mirror (tmp dir: %s)\n", tmpDir) + fmt.Fprintf(os.Stderr, "Successfully opened with local mirror (tmp dir: %s)\n", tmpDir) return db, nil } @@ -82,5 +154,7 @@ func openDatabase(path string) (*DB, error) { if strings.Contains(err.Error(), "unsupported version") { return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) } - return nil, fmt.Errorf("failed to open database from local mirror: %v", err) + + fmt.Fprintf(os.Stderr, "Failed to open with local mirror (tmp dir: %s): %v\n", tmpDir, err) + return nil, fmt.Errorf("failure: all opening strategies failed") } diff --git a/badgerdb-sampler/db_v3.go b/badgerdb-sampler/db_v3.go index 31c9b17..ea0f02e 100644 --- a/badgerdb-sampler/db_v3.go +++ b/badgerdb-sampler/db_v3.go @@ -24,17 +24,20 @@ func init() { BadgerVersion = "v3.2103.5" } -// openDatabase tries to open the database in ReadOnly mode first, -// then falls back to local mirror mode for corrupted/improperly-closed databases on FUSE +// openDatabase tries three strategies in order: +// 1. ReadOnly mode (cleanly closed databases) +// 2. Minimal read-write mode (allows log replay on original path) +// 3. Local mirror mode (FUSE workaround with mmap-safe copy) func openDatabase(path string) (*DB, error) { - // Try ReadOnly mode first (clean databases) + // Stage 1: Try read-only mode first (clean databases) + fmt.Fprintf(os.Stderr, "Opening in read-only mode...\n") opts := badger.DefaultOptions(path) opts.ReadOnly = true // Keep default logger enabled for diagnostics db, err := badger.Open(opts) if err == nil { - fmt.Fprintf(os.Stderr, "Opened database in ReadOnly mode\n") + fmt.Fprintf(os.Stderr, "Successfully opened in read-only mode\n") return db, nil } @@ -43,10 +46,82 @@ func openDatabase(path string) (*DB, error) { return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) } - fmt.Fprintf(os.Stderr, "Failed, trying local mirror workaround for FUSE...\n") + fmt.Fprintf(os.Stderr, "Failed to open in read-only mode: %v\n", err) - // Create local mirror with symlinks to avoid mmap SIGBUS errors on FUSE + // Stage 2: Try minimal read-write mode (allows log replay) + fmt.Fprintf(os.Stderr, "Opening in minimal read-write mode (allows log replay)...\n") + opts = badger.DefaultOptions(path) + opts.ReadOnly = false + opts.SyncWrites = false + opts.NumMemtables = 1 // Minimize memtable usage + opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) - v3 uses MemTableSize instead of MaxTableSize + opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) + opts.NumLevelZeroTables = 100 + opts.NumLevelZeroTablesStall = 200 + opts.NumCompactors = 0 + opts.CompactL0OnClose = false + opts.BypassLockGuard = true // Allow concurrent access + opts.DetectConflicts = false // Reduce overhead + // Keep default logger enabled for diagnostics + + db, err = badger.Open(opts) + if err == nil { + fmt.Fprintf(os.Stderr, "Successfully opened in minimal read-write mode\n") + return db, nil + } + + // Check again for version incompatibility + if strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) + } + + fmt.Fprintf(os.Stderr, "Failed to open in minimal read-write mode: %v\n", err) + + // Stage 3: Try removing corrupted memtable files and retry + fmt.Fprintf(os.Stderr, "Opening without memtable files...\n") + if backupDir, restoreMemFiles, err := removeMemtableFiles(path); backupDir != "" { + // Memtable files were removed, retry opening + opts = badger.DefaultOptions(path) + opts.ReadOnly = false + opts.SyncWrites = false + opts.NumMemtables = 1 // Minimize memtable usage + opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) - v3 uses MemTableSize instead of MaxTableSize + opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) + opts.NumLevelZeroTables = 100 + opts.NumLevelZeroTablesStall = 200 + opts.NumCompactors = 0 + opts.CompactL0OnClose = false + opts.BypassLockGuard = true // Allow concurrent access + opts.DetectConflicts = false // Reduce overhead + // Keep default logger enabled for diagnostics + + db, err = badger.Open(opts) + if err == nil { + fmt.Fprintf(os.Stderr, "Successfully opened without memtable files\n") + return db, nil + } + + // Check for version incompatibility + if strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) + } + + // Restore memtable files before continuing + fmt.Fprintf(os.Stderr, "Restoring memtable files...\n") + if restoreErr := restoreMemFiles(); restoreErr != nil { + fmt.Fprintf(os.Stderr, "Warning: Failed to restore memtable files: %v\n", restoreErr) + } else { + fmt.Fprintf(os.Stderr, "Memtable files restored\n") + } + } + + fmt.Fprintf(os.Stderr, "Failed to open without memtable files: %v\n", err) + + // Stage 4: Try local mirror workaround (FUSE compatibility) // BadgerDB v3 always uses mmap for memtables, which doesn't work on FUSE filesystems + fmt.Fprintf(os.Stderr, "Opening with local mirror (workaround for SIGBUS error on FUSE)...\n") + + // Create local mirror with symlinks to avoid mmap SIGBUS errors on FUSE tmpDir, cleanup, err := createLocalMirror(path) if err != nil { return nil, fmt.Errorf("failed to create local mirror: %w", err) @@ -55,13 +130,12 @@ func openDatabase(path string) (*DB, error) { // This is acceptable for read-only analysis tools. For long-running services, use defer cleanup() _ = cleanup - // Open from local mirror in read-write mode (needed for corrupted/improperly closed DBs) - // Unified configuration across all BadgerDB versions + // Open from local mirror with same minimal read-write settings opts = badger.DefaultOptions(tmpDir) opts.ReadOnly = false opts.SyncWrites = false opts.NumMemtables = 1 // Minimize memtable usage - opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) + opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) - v3 uses MemTableSize instead of MaxTableSize opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 @@ -73,8 +147,7 @@ func openDatabase(path string) (*DB, error) { db, err = badger.Open(opts) if err == nil { - // Successfully opened - fmt.Fprintf(os.Stderr, "Opened database from local mirror (tmp dir: %s)\n", tmpDir) + fmt.Fprintf(os.Stderr, "Successfully opened with local mirror (tmp dir: %s)\n", tmpDir) return db, nil } @@ -82,5 +155,7 @@ func openDatabase(path string) (*DB, error) { if strings.Contains(err.Error(), "unsupported version") { return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) } - return nil, fmt.Errorf("failed to open database from local mirror: %v", err) + + fmt.Fprintf(os.Stderr, "Failed to open with local mirror (tmp dir: %s): %v\n", tmpDir, err) + return nil, fmt.Errorf("failure: all opening strategies failed") } diff --git a/badgerdb-sampler/db_v4.go b/badgerdb-sampler/db_v4.go index 6d80818..04394cd 100644 --- a/badgerdb-sampler/db_v4.go +++ b/badgerdb-sampler/db_v4.go @@ -24,17 +24,20 @@ func init() { BadgerVersion = "v4.x.x" } -// openDatabase tries to open the database in ReadOnly mode first, -// then falls back to local mirror mode for corrupted/improperly-closed databases on FUSE +// openDatabase tries three strategies in order: +// 1. ReadOnly mode (cleanly closed databases) +// 2. Minimal read-write mode (allows log replay on original path) +// 3. Local mirror mode (FUSE workaround with mmap-safe copy) func openDatabase(path string) (*DB, error) { - // Try ReadOnly mode first (clean databases) + // Stage 1: Try read-only mode first (clean databases) + fmt.Fprintf(os.Stderr, "Opening in read-only mode...\n") opts := badger.DefaultOptions(path) opts.ReadOnly = true // Keep default logger enabled for diagnostics db, err := badger.Open(opts) if err == nil { - fmt.Fprintf(os.Stderr, "Opened database in ReadOnly mode\n") + fmt.Fprintf(os.Stderr, "Successfully opened in read-only mode\n") return db, nil } @@ -43,10 +46,82 @@ func openDatabase(path string) (*DB, error) { return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) } - fmt.Fprintf(os.Stderr, "Failed, trying local mirror workaround for FUSE...\n") + fmt.Fprintf(os.Stderr, "Failed to open in read-only mode: %v\n", err) - // Create local mirror with symlinks to avoid mmap SIGBUS errors on FUSE + // Stage 2: Try minimal read-write mode (allows log replay) + fmt.Fprintf(os.Stderr, "Opening in minimal read-write mode (allows log replay)...\n") + opts = badger.DefaultOptions(path) + opts.ReadOnly = false + opts.SyncWrites = false + opts.NumMemtables = 1 // Minimize memtable usage + opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) - v4 uses MemTableSize like v3 + opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) + opts.NumLevelZeroTables = 100 + opts.NumLevelZeroTablesStall = 200 + opts.NumCompactors = 0 + opts.CompactL0OnClose = false + opts.BypassLockGuard = true // Allow concurrent access + opts.DetectConflicts = false // Reduce overhead + // Keep default logger enabled for diagnostics + + db, err = badger.Open(opts) + if err == nil { + fmt.Fprintf(os.Stderr, "Successfully opened in minimal read-write mode\n") + return db, nil + } + + // Check again for version incompatibility + if strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) + } + + fmt.Fprintf(os.Stderr, "Failed to open in minimal read-write mode: %v\n", err) + + // Stage 3: Try removing corrupted memtable files and retry + fmt.Fprintf(os.Stderr, "Opening without memtable files...\n") + if backupDir, restoreMemFiles, err := removeMemtableFiles(path); backupDir != "" { + // Memtable files were removed, retry opening + opts = badger.DefaultOptions(path) + opts.ReadOnly = false + opts.SyncWrites = false + opts.NumMemtables = 1 // Minimize memtable usage + opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) - v4 uses MemTableSize like v3 + opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) + opts.NumLevelZeroTables = 100 + opts.NumLevelZeroTablesStall = 200 + opts.NumCompactors = 0 + opts.CompactL0OnClose = false + opts.BypassLockGuard = true // Allow concurrent access + opts.DetectConflicts = false // Reduce overhead + // Keep default logger enabled for diagnostics + + db, err = badger.Open(opts) + if err == nil { + fmt.Fprintf(os.Stderr, "Successfully opened without memtable files\n") + return db, nil + } + + // Check for version incompatibility + if strings.Contains(err.Error(), "unsupported version") { + return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) + } + + // Restore memtable files before continuing + fmt.Fprintf(os.Stderr, "Restoring memtable files...\n") + if restoreErr := restoreMemFiles(); restoreErr != nil { + fmt.Fprintf(os.Stderr, "Warning: Failed to restore memtable files: %v\n", restoreErr) + } else { + fmt.Fprintf(os.Stderr, "Memtable files restored\n") + } + } + + fmt.Fprintf(os.Stderr, "Failed to open without memtable files: %v\n", err) + + // Stage 4: Try local mirror workaround (FUSE compatibility) // BadgerDB v4 uses mmap for memtables similar to v3, which doesn't work on FUSE filesystems + fmt.Fprintf(os.Stderr, "Opening with local mirror (workaround for SIGBUS error on FUSE)...\n") + + // Create local mirror with symlinks to avoid mmap SIGBUS errors on FUSE tmpDir, cleanup, err := createLocalMirror(path) if err != nil { return nil, fmt.Errorf("failed to create local mirror: %w", err) @@ -55,13 +130,12 @@ func openDatabase(path string) (*DB, error) { // This is acceptable for read-only analysis tools. For long-running services, use defer cleanup() _ = cleanup - // Open from local mirror in read-write mode (needed for corrupted/improperly closed DBs) - // Unified configuration across all BadgerDB versions + // Open from local mirror with same minimal read-write settings opts = badger.DefaultOptions(tmpDir) opts.ReadOnly = false opts.SyncWrites = false opts.NumMemtables = 1 // Minimize memtable usage - opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) + opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) - v4 uses MemTableSize like v3 opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 @@ -73,8 +147,7 @@ func openDatabase(path string) (*DB, error) { db, err = badger.Open(opts) if err == nil { - // Successfully opened - fmt.Fprintf(os.Stderr, "Opened database from local mirror (tmp dir: %s)\n", tmpDir) + fmt.Fprintf(os.Stderr, "Successfully opened with local mirror (tmp dir: %s)\n", tmpDir) return db, nil } @@ -82,5 +155,7 @@ func openDatabase(path string) (*DB, error) { if strings.Contains(err.Error(), "unsupported version") { return nil, fmt.Errorf("database version incompatible with this tool. Try a different badgerdb-sampler version.\nOriginal error: %v", err) } - return nil, fmt.Errorf("failed to open database from local mirror: %v", err) + + fmt.Fprintf(os.Stderr, "Failed to open with local mirror (tmp dir: %s): %v\n", tmpDir, err) + return nil, fmt.Errorf("failure: all opening strategies failed") } diff --git a/badgerdb-sampler/main.go b/badgerdb-sampler/main.go index 87a1e84..a93529f 100644 --- a/badgerdb-sampler/main.go +++ b/badgerdb-sampler/main.go @@ -70,8 +70,8 @@ func main() { log.Printf("Warning: Could not calculate database size: %v", err) } - // Open database with two-mode strategy (corruption handling) - fmt.Printf("Opening database %s (type: %s, size: %d bytes)...\n", dbPath, dbType, dbSize) + // Open database with three-stage strategy (ReadOnly → Minimal RW → Local Mirror) + fmt.Printf("Database: %s (type: %s, size: %db)\n", dbPath, dbType, dbSize) db, err := openDatabase(dbPath) if err != nil { log.Fatalf("Failed to open database: %v", err) diff --git a/badgerdb-sampler/utils.go b/badgerdb-sampler/utils.go index 68b49f3..a4856ff 100644 --- a/badgerdb-sampler/utils.go +++ b/badgerdb-sampler/utils.go @@ -10,7 +10,7 @@ const ( // TruncateHashSize is the default hex character limit for hash truncation TruncateHashSize = 16 // TruncateLongSize is the default hex character limit for long data truncation - TruncateLongSize = 500 + TruncateLongSize = 1000 ) // extractModuleName extracts module name from MKVS leaf key and returns module:subtype description From 53ff44e51d39a0db8a10951ed3b19328ac271f3b Mon Sep 17 00:00:00 2001 From: gw0 Date: Sat, 29 Nov 2025 08:15:15 +0100 Subject: [PATCH 17/22] badgerdb-sampler: Refactor raw fields, error handling, and decoding improvements --- badgerdb-sampler/decode_consensus.go | 365 +++++++++++++++------------ badgerdb-sampler/decode_evm.go | 190 +++++++++----- badgerdb-sampler/decode_runtime.go | 206 +++++++++++---- badgerdb-sampler/main.go | 111 ++++---- badgerdb-sampler/types_consensus.go | 151 ++++++----- badgerdb-sampler/types_evm.go | 63 +++-- badgerdb-sampler/types_runtime.go | 195 +++++++++++--- badgerdb-sampler/utils.go | 106 +++++++- 8 files changed, 943 insertions(+), 444 deletions(-) diff --git a/badgerdb-sampler/decode_consensus.go b/badgerdb-sampler/decode_consensus.go index be693bf..eb7a4c1 100644 --- a/badgerdb-sampler/decode_consensus.go +++ b/badgerdb-sampler/decode_consensus.go @@ -14,8 +14,11 @@ import ( // decodeConsensusBlockstoreKey parses consensus-blockstore key and returns structured info. // See: tendermint/store/store.go for key formats (H:, P:, C:, SC:, BH:) -func decodeConsensusBlockstoreKey(key []byte) ConsensusBlockstoreKeyInfo { - info := ConsensusBlockstoreKeyInfo{} +func decodeConsensusBlockstoreKey(key []byte) *ConsensusBlockstoreKeyInfo { + info := &ConsensusBlockstoreKeyInfo{ + KeyDump: formatRawValue(key, TruncateLongLen), + KeySize: len(key), + } if len(key) < 2 { info.KeyType = "unknown" @@ -28,11 +31,8 @@ func decodeConsensusBlockstoreKey(key []byte) ConsensusBlockstoreKeyInfo { return info } - info.KeySize = len(key) - // Parse ASCII key after 0x01 prefix asciiKey := string(key[1:]) - info.KeyRaw = asciiKey // blockStore state key if asciiKey == "blockStore" { @@ -90,8 +90,11 @@ func decodeConsensusBlockstoreKey(key []byte) ConsensusBlockstoreKeyInfo { // decodeConsensusBlockstoreValue decodes protobuf value and returns structured info. // See: tendermint/proto/tendermint/store and tendermint/proto/tendermint/types -func decodeConsensusBlockstoreValue(keyType string, value []byte) ConsensusBlockstoreValueInfo { - info := ConsensusBlockstoreValueInfo{} +func decodeConsensusBlockstoreValue(keyType string, value []byte) *ConsensusBlockstoreValueInfo { + info := &ConsensusBlockstoreValueInfo{ + RawDump: formatRawValue(value, TruncateLongLen), + RawSize: len(value), + } switch keyType { case "blockstore_state": @@ -102,7 +105,7 @@ func decodeConsensusBlockstoreValue(keyType string, value []byte) ConsensusBlock ConsensusHeight: state.Height, } } else { - info.StateError = err.Error() + info.RawError = err.Error() } case "block_meta": @@ -120,7 +123,7 @@ func decodeConsensusBlockstoreValue(keyType string, value []byte) ConsensusBlock } // Format AppHash (truncate to 16 hex chars) - blockMeta.AppHash = truncateHex(meta.Header.AppHash, TruncateHashSize) + blockMeta.AppHash = formatRawValue(meta.Header.AppHash, TruncateHashLen) // Truncate ChainID chainID := meta.Header.ChainID @@ -131,7 +134,7 @@ func decodeConsensusBlockstoreValue(keyType string, value []byte) ConsensusBlock info.BlockMeta = blockMeta } else { - info.BlockMetaError = err.Error() + info.RawError = err.Error() } case "block_part": @@ -143,7 +146,7 @@ func decodeConsensusBlockstoreValue(keyType string, value []byte) ConsensusBlock ProofTotal: part.Proof.Total, } } else { - info.PartError = err.Error() + info.RawError = err.Error() } case "block_commit", "seen_commit": @@ -155,7 +158,7 @@ func decodeConsensusBlockstoreValue(keyType string, value []byte) ConsensusBlock Signatures: len(commit.Signatures), } } else { - info.CommitError = err.Error() + info.RawError = err.Error() } case "block_hash": @@ -163,17 +166,16 @@ func decodeConsensusBlockstoreValue(keyType string, value []byte) ConsensusBlock if height, err := strconv.ParseInt(string(value), 10, 64); err == nil { info.HashHeight = height } - // Note: No error field for block_hash as HashHeight is a simple int64 field } return info } // decodeConsensusEvidenceKey parses consensus-evidence key and returns structured info. -func decodeConsensusEvidenceKey(key []byte) ConsensusEvidenceKeyInfo { - info := ConsensusEvidenceKeyInfo{ +func decodeConsensusEvidenceKey(key []byte) *ConsensusEvidenceKeyInfo { + info := &ConsensusEvidenceKeyInfo{ + KeyDump: formatRawValue(key, TruncateLongLen), KeySize: len(key), - KeyHex: truncateHex(key, TruncateLongSize), } if len(key) < 2 { @@ -194,22 +196,20 @@ func decodeConsensusEvidenceKey(key []byte) ConsensusEvidenceKeyInfo { } // decodeConsensusEvidenceValue decodes evidence value and returns structured info. -// Returns (info, error) where error is a parsing error. -func decodeConsensusEvidenceValue(keyType string, value []byte) (*ConsensusEvidenceValueInfo, string) { +func decodeConsensusEvidenceValue(keyType string, value []byte) *ConsensusEvidenceValueInfo { info := &ConsensusEvidenceValueInfo{ - ValueSize: len(value), - ValueHex: truncateHex(value, TruncateLongSize), + RawDump: formatRawValue(value, TruncateLongLen), + RawSize: len(value), } if len(value) == 0 { - return info, "" + return info } // Recover from protobuf panics that occur with schema mismatches - var panicErr string defer func() { if r := recover(); r != nil { - panicErr = fmt.Sprintf("protobuf panic: %v", r) + info.RawError = fmt.Sprintf("protobuf panic: %v", r) } }() @@ -226,7 +226,7 @@ func decodeConsensusEvidenceValue(keyType string, value []byte) (*ConsensusEvide if dve.Timestamp.Unix() > 0 { info.Timestamp = dve.Timestamp.Format(time.RFC3339) } - return info, panicErr + return info } // Try to decode as LightClientAttackEvidence @@ -237,24 +237,24 @@ func decodeConsensusEvidenceValue(keyType string, value []byte) (*ConsensusEvide if lca.Timestamp.Unix() > 0 { info.Timestamp = lca.Timestamp.Format(time.RFC3339) } - return info, panicErr + return info } // If all parsing failed, show raw value info.EvidenceType = "unknown" - if panicErr != "" { - return info, panicErr + if info.RawError == "" { + info.RawError = "unable to decode evidence format" } - return info, "unable to decode evidence format" + return info } // decodeConsensusMkvsKey parses consensus-mkvs key and returns structured info. // Keys use keyformat encoding: [type_byte][data...] // See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 -func decodeConsensusMkvsKey(key []byte) ConsensusMkvsKeyInfo { - info := ConsensusMkvsKeyInfo{ +func decodeConsensusMkvsKey(key []byte) *ConsensusMkvsKeyInfo { + info := &ConsensusMkvsKeyInfo{ + KeyDump: formatRawValue(key, TruncateLongLen), KeySize: len(key), - KeyHex: truncateHex(key, TruncateLongSize), } if len(key) < 1 { @@ -269,31 +269,31 @@ func decodeConsensusMkvsKey(key []byte) ConsensusMkvsKeyInfo { switch prefixByte { case 0x00: info.KeyType = "node" - info.Hash = truncateHex(data, TruncateHashSize) + info.Hash = formatRawValue(data, TruncateHashLen) case 0x01: info.KeyType = "write_log" if len(data) >= 8 { - info.ConsensusHeight = binary.BigEndian.Uint64(data[0:8]) + info.ConsensusHeight = int64(binary.BigEndian.Uint64(data[0:8])) if len(data) >= 8+33 { - info.RootType = data[8] - info.Hash = truncateHex(data[9:9+32], TruncateHashSize) + info.RootType = fmt.Sprintf("%d", data[8]) + info.Hash = formatRawValue(data[9:9+32], TruncateHashLen) } } case 0x02: info.KeyType = "roots_metadata" if len(data) >= 8 { - info.ConsensusHeight = binary.BigEndian.Uint64(data[0:8]) + info.ConsensusHeight = int64(binary.BigEndian.Uint64(data[0:8])) } case 0x03: info.KeyType = "root_updated_nodes" if len(data) >= 8 { - info.ConsensusHeight = binary.BigEndian.Uint64(data[0:8]) + info.ConsensusHeight = int64(binary.BigEndian.Uint64(data[0:8])) if len(data) >= 8+33 { - info.RootType = data[8] - info.Hash = truncateHex(data[9:9+32], TruncateHashSize) + info.RootType = fmt.Sprintf("%d", data[8]) + info.Hash = formatRawValue(data[9:9+32], TruncateHashLen) } } @@ -303,19 +303,19 @@ func decodeConsensusMkvsKey(key []byte) ConsensusMkvsKeyInfo { case 0x05: info.KeyType = "multipart_restore_log" if len(data) >= 33 { - info.RootType = data[0] - info.Hash = truncateHex(data[1:33], TruncateHashSize) + info.RootType = fmt.Sprintf("%d", data[0]) + info.Hash = formatRawValue(data[1:33], TruncateHashLen) } else { - info.Hash = truncateHex(data, TruncateHashSize) + info.Hash = formatRawValue(data, TruncateHashLen) } case 0x06: info.KeyType = "root_node" if len(data) >= 33 { - info.RootType = data[0] - info.Hash = truncateHex(data[1:33], TruncateHashSize) + info.RootType = fmt.Sprintf("%d", data[0]) + info.Hash = formatRawValue(data[1:33], TruncateHashLen) } else { - info.Hash = truncateHex(data, TruncateHashSize) + info.Hash = formatRawValue(data, TruncateHashLen) } default: @@ -327,9 +327,10 @@ func decodeConsensusMkvsKey(key []byte) ConsensusMkvsKeyInfo { // decodeConsensusMkvsValue decodes consensus MKVS value and returns structured info. // See: _oasis-core/go/storage/mkvs/node/node.go:26-32 (prefixes), 294-309 (InternalNode), 531-537 (LeafNode) -func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueInfo { - info := ConsensusMkvsValueInfo{ - NodeSize: len(value), +func decodeConsensusMkvsValue(keyType string, value []byte) *ConsensusMkvsValueInfo { + info := &ConsensusMkvsValueInfo{ + RawDump: formatRawValue(value, TruncateLongLen), + RawSize: len(value), } if len(value) == 0 { @@ -342,9 +343,6 @@ func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueIn return info } - // Add raw node hex dump - info.NodeHex = truncateHex(value, TruncateLongSize) - // Parse MKVS node: 0x00=leaf, 0x01=internal, 0x02=nil switch value[0] { case 0x00: // LeafNode @@ -352,7 +350,7 @@ func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueIn data := value[1:] if len(data) < 2 { - info.LeafError = "key length missing" + info.RawError = "key length missing" return info } @@ -360,7 +358,7 @@ func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueIn data = data[2:] if len(data) < keySize { - info.LeafError = fmt.Sprintf("key truncated (expected %d bytes)", keySize) + info.RawError = fmt.Sprintf("key truncated (expected %d bytes)", keySize) return info } @@ -371,29 +369,29 @@ func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueIn module := decodeConsensusModulePrefix(key) leaf := &ConsensusMkvsLeafInfo{ - Module: module, + KeyDump: formatRawValue(key, TruncateLongLen), KeySize: keySize, - KeyHex: truncateHex(key, TruncateLongSize), + Module: module, } // Extract Oasis address for staking-related modules // Staking keys have format: module_prefix (1 byte) + 21-byte Oasis address if keySize == 22 { switch key[0] { - case 0x34, 0x35, 0x36, 0x37: // staking accounts, delegations, debonding, allowances + case 0x50, 0x53, 0x54: // staking accounts, delegations, debonding_delegations leaf.OasisAddress = (*(*OasisAddress)(key[1:22])).String() } } // Entity/node keys also contain addresses if keySize >= 22 { switch key[0] { - case 0x40, 0x41: // registry entities, nodes + case 0x10, 0x11: // registry entities, nodes leaf.OasisAddress = (*(*OasisAddress)(key[1:22])).String() } } if len(data) < 4 { - info.LeafError = "value length missing" + info.RawError = "value length missing" info.Leaf = leaf return info } @@ -403,18 +401,21 @@ func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueIn leaf.ValueSize = valueSize if len(data) < valueSize { - info.LeafError = fmt.Sprintf("value truncated (expected %d bytes)", valueSize) + info.RawError = fmt.Sprintf("value truncated (expected %d bytes)", valueSize) info.Leaf = leaf return info } leafValue := data[:valueSize] - leaf.ValueHex = truncateHex(leafValue, TruncateLongSize) + leaf.ValueDump = formatRawValue(leafValue, TruncateLongLen) // Try CBOR decode var decoded interface{} if err := cbor.Unmarshal(leafValue, &decoded); err == nil { - leaf.ValueFormatted = formatCBOR(decoded, valueSize) + leaf.Value = formatCBORDetailed(decoded) + leaf.ValueType = "cbor" + } else { + leaf.ValueError = err.Error() } info.Leaf = leaf @@ -425,7 +426,7 @@ func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueIn data := value[1:] if len(data) < 2 { - info.InternalError = "label bits missing" + info.RawError = "label bits missing" return info } @@ -434,7 +435,7 @@ func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueIn labelBytes := (int(labelBits) + 7) / 8 if len(data) < labelBytes+1 { - info.InternalError = "label truncated" + info.RawError = "label truncated" info.Internal = &ConsensusMkvsInternalInfo{LabelBits: labelBits} return info } @@ -445,7 +446,7 @@ func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueIn // Check for embedded leaf node or nil marker if len(data) < 1 { - info.InternalError = "missing leaf/nil marker" + info.RawError = "missing leaf/nil marker" info.Internal = internal return info } @@ -458,14 +459,14 @@ func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueIn // Skip embedded leaf: prefix(1) + keyLen(2) + key + valueLen(4) + value data = data[1:] // skip prefix if len(data) < 2 { - info.InternalError = "embedded leaf key length missing" + info.RawError = "embedded leaf key length missing" info.Internal = internal return info } keyLen := binary.LittleEndian.Uint16(data[0:2]) data = data[2:] if len(data) < int(keyLen)+4 { - info.InternalError = "embedded leaf truncated" + info.RawError = "embedded leaf truncated" info.Internal = internal return info } @@ -473,24 +474,24 @@ func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueIn valueLen := binary.LittleEndian.Uint32(data[0:4]) data = data[4:] if len(data) < int(valueLen) { - info.InternalError = "embedded leaf value truncated" + info.RawError = "embedded leaf value truncated" info.Internal = internal return info } data = data[valueLen:] // skip value } else { - info.InternalError = fmt.Sprintf("unexpected marker 0x%02x (expected 0x00 or 0x02)", data[0]) + info.RawError = fmt.Sprintf("unexpected marker 0x%02x (expected 0x00 or 0x02)", data[0]) info.Internal = internal return info } // Read left and right hashes if len(data) >= 32 { - internal.LeftHash = truncateHex(data[:32], TruncateHashSize) + internal.LeftHash = formatRawValue(data[:32], TruncateHashLen) data = data[32:] } if len(data) >= 32 { - internal.RightHash = truncateHex(data[:32], TruncateHashSize) + internal.RightHash = formatRawValue(data[:32], TruncateHashLen) } info.Internal = internal @@ -502,20 +503,65 @@ func decodeConsensusMkvsValue(keyType string, value []byte) ConsensusMkvsValueIn default: info.NodeType = "unknown" - info.NodeError = fmt.Sprintf("unknown prefix 0x%02x", value[0]) + info.RawError = fmt.Sprintf("unknown prefix 0x%02x", value[0]) return info } } // decodeConsensusModulePrefix decodes the consensus module key prefix. -// See: _oasis-core/go/consensus/tendermint/apps/*/state/state.go +// +// Prefix ranges based on Oasis Core v22.2.13: +// 0x10-0x19: Registry module +// 0x20-0x29: Roothash module +// 0x40-0x46: Beacon module +// 0x50-0x59: Staking module +// 0x60-0x63: Scheduler module +// 0x70: Keymanager module +// 0x80-0x85: Governance module +// 0xF1: Consensus parameters +// +// Source files in Oasis Core repository: +// go/consensus/tendermint/apps/registry/state/state.go (0x10-0x19) +// go/consensus/tendermint/apps/roothash/state/state.go (0x20-0x29) +// go/consensus/tendermint/apps/beacon/state/state.go (0x40-0x43, 0x45) +// go/consensus/tendermint/apps/beacon/state/state_vrf.go (0x46) +// go/consensus/tendermint/apps/staking/state/state.go (0x50-0x59) +// go/consensus/tendermint/apps/scheduler/state/state.go (0x60-0x63) +// go/consensus/tendermint/apps/keymanager/state/state.go (0x70) +// go/consensus/tendermint/apps/governance/state/state.go (0x80-0x85) +// go/consensus/tendermint/abci/state/state.go (0xF1) +// func decodeConsensusModulePrefix(key []byte) string { if len(key) == 0 { return "" } switch key[0] { + // Registry module (0x10-0x19) + // Source: _oasis-core/go/consensus/tendermint/apps/registry/state/state.go + case 0x10: + return "registry/entities" + case 0x11: + return "registry/nodes" + case 0x12: + return "registry/node_by_entity" + case 0x13: + return "registry/runtimes" + case 0x14: + return "registry/node_by_consensus_address" + case 0x15: + return "registry/node_status" + case 0x16: + return "registry/params" + case 0x17: + return "registry/key_map" + case 0x18: + return "registry/suspended_runtimes" + case 0x19: + return "registry/runtime_by_entity" + // Roothash module (0x20-0x29) + // Source: _oasis-core/go/consensus/tendermint/apps/roothash/state/state.go case 0x20: return "roothash/runtime_state" case 0x21: @@ -534,86 +580,97 @@ func decodeConsensusModulePrefix(key []byte) string { return "roothash/incoming_msg_queue_meta" case 0x29: return "roothash/incoming_msg_queue" - // Staking module (0x30-0x3F) - case 0x30: - return "staking/total_supply" - case 0x31: - return "staking/common_pool" - case 0x32: - return "staking/last_block_fees" - case 0x33: - return "staking/governance_deposits" - case 0x34: - return "staking/accounts" - case 0x35: - return "staking/delegations" - case 0x36: - return "staking/debonding_delegations" - case 0x37: - return "staking/allowances" - case 0x38: - return "staking/params" - // Registry module (0x40-0x4F) + + // Beacon module (0x40-0x46) + // Source: _oasis-core/go/consensus/tendermint/apps/beacon/state/*.go case 0x40: - return "registry/entities" + return "beacon/epoch_current" case 0x41: - return "registry/nodes" + return "beacon/epoch_future" case 0x42: - return "registry/node_by_consensus" + return "beacon/beacon" case 0x43: - return "registry/runtimes" + return "beacon/params" case 0x44: - return "registry/suspended_runtimes" + return "beacon/pvss_state_deprecated" case 0x45: - return "registry/params" + return "beacon/epoch_pending_mock" case 0x46: - return "registry/node_status" - // Scheduler module (0x50-0x5F) + return "beacon/vrf_state" + + // Staking module (0x50-0x59) + // Source: _oasis-core/go/consensus/tendermint/apps/staking/state/state.go case 0x50: - return "scheduler/params" + return "staking/accounts" case 0x51: - return "scheduler/committees" + return "staking/total_supply" case 0x52: - return "scheduler/validators" - // Governance module (0x60-0x6F) + return "staking/common_pool" + case 0x53: + return "staking/delegations" + case 0x54: + return "staking/debonding_delegations" + case 0x55: + return "staking/debonding_queue" + case 0x56: + return "staking/params" + case 0x57: + return "staking/last_block_fees" + case 0x58: + return "staking/epoch_signing" + case 0x59: + return "staking/governance_deposits" + + // Scheduler module (0x60-0x63) + // Source: _oasis-core/go/consensus/tendermint/apps/scheduler/state/state.go case 0x60: - return "governance/params" + return "scheduler/committees" case 0x61: - return "governance/proposals" + return "scheduler/validators_current" case 0x62: - return "governance/active_proposals" + return "scheduler/validators_pending" case 0x63: - return "governance/votes" - case 0x64: - return "governance/pending_upgrades" - // Beacon module (0x70-0x7F) + return "scheduler/params" + + // Keymanager module (0x70) + // Source: _oasis-core/go/consensus/tendermint/apps/keymanager/state/state.go case 0x70: - return "beacon/params" - case 0x71: - return "beacon/future_epoch" - case 0x72: - return "beacon/epoch" - case 0x73: - return "beacon/pvss_state" - // Keymanager module (0x80-0x8F) - case 0x80: return "keymanager/status" + + // Governance module (0x80-0x85) + // Source: _oasis-core/go/consensus/tendermint/apps/governance/state/state.go + case 0x80: + return "governance/next_proposal_id" case 0x81: - return "keymanager/params" - // Consensus parameters + return "governance/proposals" + case 0x82: + return "governance/active_proposals" + case 0x83: + return "governance/votes" + case 0x84: + return "governance/pending_upgrades" + case 0x85: + return "governance/params" + + // Consensus parameters (0xF1) + // Source: _oasis-core/go/consensus/tendermint/abci/state/state.go case 0xF1: return "consensus/params" + default: if key[0] >= 'a' && key[0] <= 'z' { return extractModuleName(key) } - return fmt.Sprintf("0x%02x", key[0]) + return fmt.Sprintf("unknown:0x%02x", key[0]) } } // decodeConsensusStateKey parses consensus-state key and returns structured info. -func decodeConsensusStateKey(key []byte) ConsensusStateKeyInfo { - info := ConsensusStateKeyInfo{} +func decodeConsensusStateKey(key []byte) *ConsensusStateKeyInfo { + info := &ConsensusStateKeyInfo{ + KeyDump: formatRawValue(key, TruncateLongLen), + KeySize: len(key), + } if len(key) < 2 { info.KeyType = "unknown" @@ -626,11 +683,8 @@ func decodeConsensusStateKey(key []byte) ConsensusStateKeyInfo { return info } - info.KeySize = len(key) - // Parse ASCII key after 0x01 prefix decoded := string(key[1:]) - info.KeyHex = truncateHex(key, TruncateLongSize) // Check if it's printable ASCII if !isPrintableASCII(decoded) { @@ -683,10 +737,10 @@ func decodeConsensusStateKey(key []byte) ConsensusStateKeyInfo { } // decodeConsensusStateValue decodes state value and returns structured info. -func decodeConsensusStateValue(keyType string, value []byte) ConsensusStateValueInfo { - info := ConsensusStateValueInfo{ - ValueSize: len(value), - ValueHex: truncateHex(value, TruncateLongSize), +func decodeConsensusStateValue(keyType string, value []byte) *ConsensusStateValueInfo { + info := &ConsensusStateValueInfo{ + RawDump: formatRawValue(value, TruncateLongLen), + RawSize: len(value), } if len(value) == 0 { @@ -697,25 +751,8 @@ func decodeConsensusStateValue(keyType string, value []byte) ConsensusStateValue defer func() { if r := recover(); r != nil { // Protobuf panic occurred (likely schema mismatch) - errMsg := fmt.Sprintf("protobuf panic: %v", r) - // Set appropriate error field based on key type - switch keyType { - case "abci_responses": - if info.ABCIResponseError == "" { - info.ABCIResponseError = errMsg - } - case "consensus_params": - if info.ConsensusParamsError == "" { - info.ConsensusParamsError = errMsg - } - case "validators": - if info.ValidatorsError == "" { - info.ValidatorsError = errMsg - } - case "state", "genesis": - if info.StateError == "" { - info.StateError = errMsg - } + if info.RawError == "" { + info.RawError = fmt.Sprintf("protobuf panic: %v", r) } } }() @@ -727,7 +764,7 @@ func decodeConsensusStateValue(keyType string, value []byte) ConsensusStateValue if err := proto.Unmarshal(value, &resp); err == nil { // Validate that unmarshal actually decoded meaningful data if resp.DeliverTxs == nil && resp.BeginBlock == nil && resp.EndBlock == nil { - info.ABCIResponseError = "protobuf unmarshal succeeded but all fields are nil (schema mismatch)" + info.RawError = "protobuf unmarshal succeeded but all fields are nil (schema mismatch)" } else { abciResponseInfo := &ConsensusABCIResponseInfo{} @@ -827,7 +864,7 @@ func decodeConsensusStateValue(keyType string, value []byte) ConsensusStateValue info.ABCIResponse = abciResponseInfo } } else { - info.ABCIResponseError = fmt.Sprintf("failed to decode ABCI responses: %v", err) + info.RawError = fmt.Sprintf("failed to decode ABCI responses: %v", err) } case "consensus_params": @@ -835,12 +872,12 @@ func decodeConsensusStateValue(keyType string, value []byte) ConsensusStateValue if err := proto.Unmarshal(value, ¶ms); err == nil { // Validate that unmarshal decoded meaningful data if params.Block == nil && params.Evidence == nil && params.Validator == nil && params.Version == nil { - info.ConsensusParamsError = "protobuf unmarshal succeeded but all fields are nil (schema mismatch)" + info.RawError = "protobuf unmarshal succeeded but all fields are nil (schema mismatch)" } else { info.ConsensusParams = ¶ms } } else { - info.ConsensusParamsError = fmt.Sprintf("failed to decode consensus params: %v", err) + info.RawError = fmt.Sprintf("failed to decode consensus params: %v", err) } case "validators": @@ -848,12 +885,12 @@ func decodeConsensusStateValue(keyType string, value []byte) ConsensusStateValue if err := proto.Unmarshal(value, &valSet); err == nil { // Validate that unmarshal decoded meaningful data if len(valSet.Validators) == 0 && valSet.Proposer == nil && valSet.TotalVotingPower == 0 { - info.ValidatorsError = "protobuf unmarshal succeeded but all fields are empty/nil (schema mismatch)" + info.RawError = "protobuf unmarshal succeeded but all fields are empty/nil (schema mismatch)" } else { info.ConsensusValidators = &valSet } } else { - info.ValidatorsError = fmt.Sprintf("failed to decode validator set: %v", err) + info.RawError = fmt.Sprintf("failed to decode validator set: %v", err) } case "state": @@ -861,19 +898,19 @@ func decodeConsensusStateValue(keyType string, value []byte) ConsensusStateValue if err := proto.Unmarshal(value, &state); err == nil { // Validate that unmarshal decoded meaningful data if state.ChainID == "" && state.LastBlockHeight == 0 && state.InitialHeight == 0 { - info.StateError = "protobuf unmarshal succeeded but all fields are empty/zero (schema mismatch)" + info.RawError = "protobuf unmarshal succeeded but all fields are empty/zero (schema mismatch)" } else { info.ConsensusState = &state } } else { - info.StateError = fmt.Sprintf("failed to decode state: %v", err) + info.RawError = fmt.Sprintf("failed to decode state: %v", err) } case "genesis": if value[0] != '{' { - info.StateError = "genesis document not in expected JSON format" + info.RawError = "genesis document not in expected JSON format" } - // Genesis is JSON, ValueRaw shows hex preview (already set above) + // Genesis is JSON, RawDump shows hex preview (already set above) } return info @@ -894,7 +931,7 @@ func decodeConsensusTransaction(rawTx []byte) (ConsensusTransactionInfo, error) } // Extract signer public key - info.Signer = truncateHex(signedTx.Signature.PublicKey[:], TruncateLongSize) + info.Signer = formatRawValue(signedTx.Signature.PublicKey[:], TruncateLongLen) // Decode inner Transaction from the blob var tx cborConsensusInnerTransaction @@ -913,7 +950,7 @@ func decodeConsensusTransaction(rawTx []byte) (ConsensusTransactionInfo, error) // Decode body based on method if len(tx.Body) > 0 { - info.BodyHex = truncateHex(tx.Body, TruncateLongSize) + info.BodyHex = formatRawValue(tx.Body, TruncateLongLen) info.BodySize = len(tx.Body) switch tx.Method { @@ -1022,7 +1059,7 @@ func decodeConsensusEvent(event tmEvent) ([]ConsensusEventInfo, error) { } // Populate raw body fields - info.BodyHex = truncateHex(cborData, TruncateLongSize) + info.BodyHex = formatRawValue(cborData, TruncateLongLen) info.BodySize = len(cborData) // Decode body based on event kind diff --git a/badgerdb-sampler/decode_evm.go b/badgerdb-sampler/decode_evm.go index edc6156..7c53c14 100644 --- a/badgerdb-sampler/decode_evm.go +++ b/badgerdb-sampler/decode_evm.go @@ -3,6 +3,7 @@ package main import ( "encoding/binary" + "encoding/hex" "fmt" "github.com/fxamacker/cbor/v2" @@ -41,27 +42,27 @@ func decodeEVMData(module string, key []byte, value []byte) *EVMDataInfo { case 0x01: // CODES: evm + 0x01 + H160 (address) evmInfo.StorageType = "code" if len(data) >= 20 { - evmInfo.Address = truncateHex0x(data[:20], TruncateLongSize) + evmInfo.Address = "0x" + hex.EncodeToString(data[:20]) data = data[20:] } // Value is contract bytecode evmInfo.CodeSize = len(value) if len(value) > 0 { - evmInfo.CodeHex = truncateHex(value, TruncateLongSize) + evmInfo.CodeHex = formatRawValue(value, TruncateLongLen) } case 0x02: // STORAGES: evm + 0x02 + H160 (address) + H256 (slot) evmInfo.StorageType = "storage" if len(data) >= 20 { - evmInfo.Address = truncateHex0x(data[:20], TruncateLongSize) + evmInfo.Address = "0x" + hex.EncodeToString(data[:20]) data = data[20:] if len(data) >= 32 { - evmInfo.StorageSlot = truncateHex0x(data[:32], TruncateHashSize) + evmInfo.StorageSlot = formatRawValue(data[:32], TruncateHashLen) } } // Value is H256 storage value if len(value) == 32 { - evmInfo.StorageValue = truncateHex0x(value, TruncateLongSize) + evmInfo.StorageValue = formatRawValue(value, TruncateLongLen) } case 0x03: // BLOCK_HASHES: evm + 0x03 + RuntimeHeight (uint64 BE) @@ -71,16 +72,16 @@ func decodeEVMData(module string, key []byte, value []byte) *EVMDataInfo { } // Value is H256 block hash if len(value) == 32 { - evmInfo.BlockHash = truncateHex0x(value, TruncateHashSize) + evmInfo.BlockHash = formatRawValue(value, TruncateHashLen) } case 0x04: // CONFIDENTIAL_STORAGES: evm + 0x04 + H160 (address) + H256 (slot) evmInfo.StorageType = "confidential_storage" if len(data) >= 20 { - evmInfo.Address = truncateHex0x(data[:20], TruncateLongSize) + evmInfo.Address = "0x" + hex.EncodeToString(data[:20]) data = data[20:] if len(data) >= 32 { - evmInfo.StorageSlot = truncateHex0x(data[:32], TruncateHashSize) + evmInfo.StorageSlot = formatRawValue(data[:32], TruncateHashLen) } } // Value is encrypted - we can only show size @@ -98,29 +99,35 @@ func decodeEVMData(module string, key []byte, value []byte) *EVMDataInfo { // decodeEVMEvent decodes an EVM Log event from CBOR-encoded value. // See: _oasis-sdk/runtime-sdk/modules/evm/src/lib.rs:253-263 // Returns (info, error) where error is a parsing error, not an execution error. -func decodeEVMEvent(value []byte) (*EVMEventInfo, string) { - info := &EVMEventInfo{} +func decodeEVMEvent(value []byte) *EVMEventInfo { + info := &EVMEventInfo{ + RawDump: formatRawValue(value, TruncateLongLen), + RawSize: len(value), + } // Value is CBOR: [event_code, {address, topics, data}] var eventWrapper []interface{} if err := cbor.Unmarshal(value, &eventWrapper); err != nil { - return nil, err.Error() + info.RawError = fmt.Sprintf("cbor unmarshal failed: %v", err) + return info } if len(eventWrapper) < 2 { - return nil, "invalid format" + info.RawError = "invalid format: insufficient fields" + return info } eventData, ok := eventWrapper[1].(map[interface{}]interface{}) if !ok { - return nil, "invalid format" + info.RawError = "invalid format: element[1] not a map" + return info } // Extract address (H160) if addrBytes, ok := eventData["address"].([]byte); ok && len(addrBytes) == 20 { - info.Address = truncateHex0x(addrBytes, TruncateLongSize) + info.Address = "0x" + hex.EncodeToString(addrBytes) } else { - // Invalid address, but continue decoding - return partial data with error - return info, "invalid address (partial decode)" + info.RawError = "invalid address" + return info } // Extract topics (Vec) @@ -128,11 +135,11 @@ func decodeEVMEvent(value []byte) (*EVMEventInfo, string) { info.TopicCount = len(topicsArray) for i, topic := range topicsArray { if topicBytes, ok := topic.([]byte); ok && len(topicBytes) == 32 { - topicHex := truncateHex0x(topicBytes, TruncateHashSize) + topicHex := "0x" + hex.EncodeToString(topicBytes) info.Topics = append(info.Topics, topicHex) if i == 0 { info.EventHash = topicHex - if sig, found := EVMEventSignatures[topicHex[:len(topicHex)]]; found { + if sig, found := EVMEventSignatures[topicHex]; found { info.EventSignature = sig } } @@ -143,30 +150,43 @@ func decodeEVMEvent(value []byte) (*EVMEventInfo, string) { // Extract data if dataBytes, ok := eventData["data"].([]byte); ok { info.DataSize = len(dataBytes) - info.DataHex = truncateHex(dataBytes, TruncateLongSize) + info.DataHex = formatRawValue(dataBytes, TruncateLongLen) } - return info, "" + return info } // decodeEVMTxInput decodes EVM transaction input artifacts. -// Returns (info, error) where error is a parsing error. -func decodeEVMTxInput(key []byte, value []byte) (*EVMTxInputInfo, string) { +func decodeEVMTxInput(key []byte, value []byte) *EVMTxInputInfo { info := &EVMTxInputInfo{ - TxInputSize: len(value), - TxInputHex: truncateHex(value, TruncateLongSize), + RawDump: formatRawValue(value, TruncateLongLen), + RawSize: len(value), } if len(key) >= 33 { - info.TxHash = truncateHex0x(key[1:33], TruncateHashSize) + info.TxHash = formatRawValue(key[1:33], TruncateHashLen) } var ia cborRuntimeInputArtifacts if err := cbor.Unmarshal(value, &ia); err != nil { - return nil, fmt.Sprintf("failed to unmarshal RuntimeInputArtifacts: %v", err) + info.RawError = fmt.Sprintf("failed to unmarshal RuntimeInputArtifacts: %v", err) + return info } info.BatchOrder = ia.BatchOrder + // Attempt 0: Variant wrapper format [version, [variant_tag, data]] + var wrapperArray []interface{} + if err := cbor.Unmarshal(ia.Input, &wrapperArray); err == nil && len(wrapperArray) >= 2 { + if nestedArray, ok := wrapperArray[1].([]interface{}); ok && len(nestedArray) >= 2 { + // Extract the actual transaction data (second element of nested array) + if txDataBytes, err := cbor.Marshal(nestedArray[1]); err == nil { + // Recursively decode the unwrapped transaction + ia.Input = txDataBytes + return decodeEVMTxInput(key, value) + } + } + } + // Try decoding as array format first (old runtime version) var callArray cborRuntimeCallArrayFormat if err := cbor.Unmarshal(ia.Input, &callArray); err == nil { @@ -174,9 +194,9 @@ func decodeEVMTxInput(key []byte, value []byte) (*EVMTxInputInfo, string) { // Decode EVM call methods that have body structure if callArray.Method == "evm.Call" || callArray.Method == "evm.Create" || callArray.Method == "evm.SimulateCall" || callArray.Method == "evm.EstimateGas" { isCreate := callArray.Method == "evm.Create" - info.EVMCall = decodeEVMCallBodyFromCBOR(callArray.Body, isCreate) + info.EVMTx = decodeEVMTransactionFromCBOR(callArray.Body, isCreate) } - return info, "" + return info } // Try decoding as map format (newer runtime version) @@ -186,9 +206,9 @@ func decodeEVMTxInput(key []byte, value []byte) (*EVMTxInputInfo, string) { // Decode EVM call methods that have body structure if callMap.Method == "evm.Call" || callMap.Method == "evm.Create" || callMap.Method == "evm.SimulateCall" || callMap.Method == "evm.EstimateGas" { isCreate := callMap.Method == "evm.Create" - info.EVMCall = decodeEVMCallBodyFromCBOR(callMap.Body, isCreate) + info.EVMTx = decodeEVMTransactionFromCBOR(callMap.Body, isCreate) } - return info, "" + return info } // Fallback: try generic array decode (for compatibility with other runtime versions) @@ -196,7 +216,8 @@ func decodeEVMTxInput(key []byte, value []byte) (*EVMTxInputInfo, string) { if err := cbor.Unmarshal(ia.Input, &callArrayGeneric); err == nil { // Array format: [format, method, body, ...] or [format, [method, body, ...]] if len(callArrayGeneric) < 2 { - return info, fmt.Sprintf("array format: insufficient fields (len=%d), expected at least 2", len(callArrayGeneric)) + info.RawError = "array format: insufficient fields" + return info } // Extract method and body - handle both flat and nested array formats @@ -212,7 +233,8 @@ func decodeEVMTxInput(key []byte, value []byte) (*EVMTxInputInfo, string) { } else if nestedArray, ok := callArrayGeneric[1].([]interface{}); ok { // Nested format: [format, [method, body, ...]] or [format, [{method: "...", body: ...}]] if len(nestedArray) < 1 { - return info, "nested array format: element[1] is empty array" + info.RawError = "nested array format: empty" + return info } if methodVal, ok := nestedArray[0].(string); ok { // Nested array format: [format, [method, body, ...]] @@ -228,21 +250,24 @@ func decodeEVMTxInput(key []byte, value []byte) (*EVMTxInputInfo, string) { bodyBytes = bodyVal } } else { - return info, "nested map format: method field missing or invalid" + info.RawError = "nested map format: method missing" + return info } } else { - return info, fmt.Sprintf("nested array format: element[1][0] is not a valid method string or map (type: %T)", nestedArray[0]) + info.RawError = "nested array format: invalid element type" + return info } } else { - return info, fmt.Sprintf("array format: element[1] is not a valid method string or array (type: %T)", callArrayGeneric[1]) + info.RawError = "array format: invalid element[1] type" + return info } info.Method = method // Decode EVM call methods that have body structure if bodyBytes != nil && (method == "evm.Call" || method == "evm.Create" || method == "evm.SimulateCall" || method == "evm.EstimateGas") { - info.EVMCall = decodeEVMCallBodyFromCBOR(bodyBytes, method == "evm.Create") + info.EVMTx = decodeEVMTransactionFromCBOR(bodyBytes, method == "evm.Create") } - return info, "" + return info } // Final fallback: try legacy map[interface{}]interface{} decode @@ -255,44 +280,77 @@ func decodeEVMTxInput(key []byte, value []byte) (*EVMTxInputInfo, string) { isCreate := methodVal == "evm.Create" // Extract body bytes and decode if bodyBytes, ok := call["body"].([]byte); ok { - info.EVMCall = decodeEVMCallBodyFromCBOR(bodyBytes, isCreate) + info.EVMTx = decodeEVMTransactionFromCBOR(bodyBytes, isCreate) } } - return info, "" + return info } - return info, "map format: method field missing or invalid" + info.RawError = "map format: method missing" + return info } // All decode attempts failed - return info, "all formats failed" + info.RawError = "all formats failed" + return info } -// decodeEVMCallBodyFromCBOR decodes EVM call/create body from raw CBOR bytes. -func decodeEVMCallBodyFromCBOR(bodyBytes []byte, isCreate bool) *EVMCallInfo { - info := &EVMCallInfo{} +// decodeEVMTransactionFromCBOR decodes EVM transaction from raw CBOR bytes. +// See: _oasis-sdk/runtime-sdk/modules/evm/src/types.rs for transaction structure +func decodeEVMTransactionFromCBOR(bodyBytes []byte, isCreate bool) *EVMTransactionInfo { + info := &EVMTransactionInfo{ + RawDump: formatRawValue(bodyBytes, TruncateLongLen), + RawSize: len(bodyBytes), + } + if isCreate { info.Type = "create" } else { info.Type = "call" } + // Try to unmarshal as map var bodyMap map[interface{}]interface{} if err := cbor.Unmarshal(bodyBytes, &bodyMap); err != nil { + info.RawError = fmt.Sprintf("cbor unmarshal failed: %v", err) return info } - // Extract address (calls only) + // Extract from address (20 bytes) + if fromBytes, ok := bodyMap["from"].([]byte); ok && len(fromBytes) == 20 { + info.From = "0x" + hex.EncodeToString(fromBytes) + } + + // Extract to address (20 bytes, calls only) if !isCreate { - if addrBytes, ok := bodyMap["address"].([]byte); ok && len(addrBytes) == 20 { - info.Address = truncateHex0x(addrBytes, TruncateLongSize) + if toBytes, ok := bodyMap["address"].([]byte); ok && len(toBytes) == 20 { + info.To = "0x" + hex.EncodeToString(toBytes) + } else if toBytes, ok := bodyMap["to"].([]byte); ok && len(toBytes) == 20 { + info.To = "0x" + hex.EncodeToString(toBytes) } } - // Extract value + // Extract value (U256) if valueBytes, ok := bodyMap["value"].([]byte); ok { info.Value = formatU256(valueBytes) } + // Extract gas_limit (u64) + if gasLimit, ok := bodyMap["gas_limit"].(uint64); ok { + info.GasLimit = gasLimit + } else if gasLimit, ok := bodyMap["gas"].(uint64); ok { + info.GasLimit = gasLimit + } + + // Extract gas_price (U256) + if gasPriceBytes, ok := bodyMap["gas_price"].([]byte); ok { + info.GasPrice = formatU256(gasPriceBytes) + } + + // Extract nonce (u64) + if nonce, ok := bodyMap["nonce"].(uint64); ok { + info.Nonce = nonce + } + // Extract data/init_code dataKey := "data" if isCreate { @@ -300,58 +358,58 @@ func decodeEVMCallBodyFromCBOR(bodyBytes []byte, isCreate bool) *EVMCallInfo { } if dataBytes, ok := bodyMap[dataKey].([]byte); ok { info.DataSize = len(dataBytes) - info.DataHex = truncateHex(dataBytes, TruncateLongSize) + info.DataDump = formatRawValue(dataBytes, TruncateLongLen) } return info } // decodeEVMTxOutput decodes EVM transaction output artifacts. -// Returns (info, error) where error is a parsing error. -// Execution errors (transaction failures) are stored in info.Error field. -func decodeEVMTxOutput(value []byte) (*EVMTxOutputInfo, string) { +// Execution errors (transaction failures) are stored in info.ErrorExecution field. +func decodeEVMTxOutput(value []byte) *EVMTxOutputInfo { info := &EVMTxOutputInfo{ - TxOutputSize: len(value), - TxOutputHex: truncateHex(value, TruncateLongSize), + RawDump: formatRawValue(value, TruncateLongLen), + RawSize: len(value), } var oa cborRuntimeOutputArtifacts if err := cbor.Unmarshal(value, &oa); err != nil { - return nil, err.Error() + info.RawError = fmt.Sprintf("failed to unmarshal RuntimeOutputArtifacts: %v", err) + return info } if len(oa.Output) == 0 { - info.Success = true - return info, "" + info.SuccessExecution = true + return info } // Try to decode as CBOR CallResult var result map[interface{}]interface{} if err := cbor.Unmarshal(oa.Output, &result); err != nil { // Raw bytes - success - info.Success = true + info.SuccessExecution = true info.ResultSize = len(oa.Output) - info.ResultHex = truncateHex(oa.Output, TruncateLongSize) - return info, "" + info.ResultDump = formatRawValue(oa.Output, TruncateLongLen) + return info } // Check success/failure if okVal, exists := result["ok"]; exists { - info.Success = true + info.SuccessExecution = true if okBytes, ok := okVal.([]byte); ok { info.ResultSize = len(okBytes) - info.ResultHex = truncateHex(okBytes, TruncateLongSize) + info.ResultDump = formatRawValue(okBytes, TruncateLongLen) } } else if failVal, exists := result["fail"]; exists { - info.Success = false + info.SuccessExecution = false if failMap, ok := failVal.(map[interface{}]interface{}); ok { if msgVal, ok := failMap["message"].(string); ok { - info.Error = msgVal // Execution error, keep in struct + info.ErrorExecution = msgVal // Execution error, keep in struct } else { - info.Error = fmt.Sprintf("%v", failMap) + info.ErrorExecution = fmt.Sprintf("%v", failMap) } } } - return info, "" + return info } diff --git a/badgerdb-sampler/decode_runtime.go b/badgerdb-sampler/decode_runtime.go index 2fe5e38..e3ca621 100644 --- a/badgerdb-sampler/decode_runtime.go +++ b/badgerdb-sampler/decode_runtime.go @@ -3,6 +3,8 @@ package main import ( "encoding/binary" "fmt" + "math/big" + "strings" "time" "github.com/fxamacker/cbor/v2" @@ -11,8 +13,11 @@ import ( // decodeRuntimeMkvsKey parses runtime-mkvs key and returns structured info. // Keys use keyformat encoding: [type_byte][data...] // See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 -func decodeRuntimeMkvsKey(key []byte) RuntimeMkvsKeyInfo { - info := RuntimeMkvsKeyInfo{} +func decodeRuntimeMkvsKey(key []byte) *RuntimeMkvsKeyInfo { + info := &RuntimeMkvsKeyInfo{ + KeyDump: formatRawValue(key, TruncateLongLen), + KeySize: len(key), + } if len(key) < 1 { info.KeyType = "unknown" @@ -26,7 +31,7 @@ func decodeRuntimeMkvsKey(key []byte) RuntimeMkvsKeyInfo { switch prefixByte { case 0x00: info.KeyType = "node" - info.Hash = truncateHex(data, TruncateHashSize) + info.Hash = formatRawValue(data, TruncateHashLen) case 0x01: info.KeyType = "write_log" if len(data) >= 8 { @@ -53,8 +58,11 @@ func decodeRuntimeMkvsKey(key []byte) RuntimeMkvsKeyInfo { // decodeRuntimeMkvsValue decodes runtime MKVS value and returns structured info. // See: _oasis-core/go/storage/mkvs/node/node.go:26-32 (prefixes), 294-309 (InternalNode), 531-537 (LeafNode) -func decodeRuntimeMkvsValue(keyType string, value []byte) RuntimeMkvsNodeInfo { - info := RuntimeMkvsNodeInfo{NodeSize: len(value)} +func decodeRuntimeMkvsValue(keyType string, value []byte) *RuntimeMkvsValueInfo { + info := &RuntimeMkvsValueInfo{ + RawDump: formatRawValue(value, TruncateLongLen), + RawSize: len(value), + } if len(value) == 0 { info.NodeType = "empty" @@ -66,9 +74,6 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) RuntimeMkvsNodeInfo { return info } - // Add raw node hex dump - info.NodeHex = truncateHex(value, TruncateLongSize) - // Parse MKVS node: 0x00=leaf, 0x01=internal, 0x02=nil switch value[0] { case 0x00: // LeafNode: [2-byte keySize LE][key][4-byte valueSize LE][value] @@ -76,7 +81,7 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) RuntimeMkvsNodeInfo { data := value[1:] if len(data) < 2 { - info.LeafError = "key length missing" + info.RawError = "key length missing" return info } @@ -84,7 +89,7 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) RuntimeMkvsNodeInfo { data = data[2:] if len(data) < keySize { - info.LeafError = fmt.Sprintf("key truncated (expected %d bytes)", keySize) + info.RawError = fmt.Sprintf("key truncated (expected %d bytes)", keySize) return info } @@ -97,27 +102,25 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) RuntimeMkvsNodeInfo { leaf := &RuntimeMkvsLeafInfo{ Module: module, KeySize: keySize, - KeyHex: truncateHex(key, TruncateLongSize), + KeyDump: formatRawValue(key, TruncateLongLen), } if len(data) < 4 { - info.LeafError = "value length missing" + info.RawError = "value length missing" info.Leaf = leaf return info } valueSize := int(binary.LittleEndian.Uint32(data[0:4])) data = data[4:] - leaf.ValueSize = valueSize if len(data) < valueSize { - info.LeafError = fmt.Sprintf("value truncated (expected %d bytes)", valueSize) + info.RawError = fmt.Sprintf("value truncated (expected %d bytes)", valueSize) info.Leaf = leaf return info } leafValue := data[:valueSize] - leaf.ValueHex = truncateHex(leafValue, TruncateLongSize) leaf.Value = decodeRuntimeLeafValue(module, key, leafValue) info.Leaf = leaf return info @@ -127,7 +130,7 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) RuntimeMkvsNodeInfo { data := value[1:] if len(data) < 2 { - info.InternalError = "label bits missing" + info.RawError = "label bits missing" return info } @@ -136,7 +139,7 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) RuntimeMkvsNodeInfo { labelBytes := (int(labelBits) + 7) / 8 if len(data) < labelBytes+1 { - info.InternalError = "label truncated" + info.RawError = "label truncated" info.Internal = &RuntimeMkvsInternalInfo{LabelBits: labelBits} return info } @@ -151,11 +154,11 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) RuntimeMkvsNodeInfo { data = data[1:] // skip nil marker // Extract child hashes if len(data) >= 32 { - internal.LeftHash = truncateHex(data[:32], TruncateHashSize) + internal.LeftHash = formatRawValue(data[:32], TruncateHashLen) data = data[32:] } if len(data) >= 32 { - internal.RightHash = truncateHex(data[:32], TruncateHashSize) + internal.RightHash = formatRawValue(data[:32], TruncateHashLen) } } @@ -168,7 +171,7 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) RuntimeMkvsNodeInfo { default: info.NodeType = "unknown" - info.NodeError = fmt.Sprintf("unknown prefix 0x%02x", value[0]) + info.RawError = fmt.Sprintf("unknown prefix 0x%02x", value[0]) return info } } @@ -179,8 +182,11 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) RuntimeMkvsNodeInfo { // - 0x01: metadata key (1 byte) // - 0x02 + uint64: block key (9 bytes) - round number in big-endian // - 0x03 + uint64: round_results key (9 bytes) - round number in big-endian -func decodeRuntimeHistoryKey(key []byte) RuntimeHistoryKeyInfo { - info := RuntimeHistoryKeyInfo{} +func decodeRuntimeHistoryKey(key []byte) *RuntimeHistoryKeyInfo { + info := &RuntimeHistoryKeyInfo{ + KeyDump: formatRawValue(key, TruncateLongLen), + KeySize: len(key), + } if len(key) == 0 { info.KeyType = "unknown" @@ -191,7 +197,7 @@ func decodeRuntimeHistoryKey(key []byte) RuntimeHistoryKeyInfo { case 0x01: info.KeyType = "metadata" if len(key) > 1 { - info.ExtraData = truncateHex(key[1:], TruncateLongSize) + info.ExtraData = formatRawValue(key[1:], TruncateLongLen) } case 0x02: @@ -217,19 +223,13 @@ func decodeRuntimeHistoryKey(key []byte) RuntimeHistoryKeyInfo { // See: _oasis-core/go/runtime/history/db.go:34-44 (metadata) // See: _oasis-core/go/roothash/api/api.go:402-409 (AnnotatedBlock) // See: _oasis-core/go/roothash/api/results.go:5-17 (RoundResults) -func decodeRuntimeHistoryValue(keyType string, value []byte) RuntimeHistoryValueInfo { - info := RuntimeHistoryValueInfo{} +func decodeRuntimeHistoryValue(keyType string, value []byte) *RuntimeHistoryValueInfo { + info := &RuntimeHistoryValueInfo{ + RawDump: formatRawValue(value, TruncateLongLen), + RawSize: len(value), + } if len(value) == 0 { - // General error when we don't know which child yet - switch keyType { - case "metadata": - info.MetadataError = "empty value" - case "block": - info.BlockError = "empty value" - case "round_results": - info.RoundResultsError = "empty value" - } return info } @@ -237,12 +237,12 @@ func decodeRuntimeHistoryValue(keyType string, value []byte) RuntimeHistoryValue case "metadata": var meta cborRuntimeHistoryMetadata if err := cbor.Unmarshal(value, &meta); err != nil { - info.MetadataError = err.Error() + info.RawError = err.Error() return info } info.Metadata = &RuntimeHistoryMetadataInfo{ Version: meta.Version, - RuntimeID: truncateHex(meta.RuntimeID, TruncateLongSize), + RuntimeID: formatRawValue(meta.RuntimeID, TruncateLongLen), LastRuntimeHeight: meta.LastRound, LastConsensusHeight: meta.LastConsensusHeight, } @@ -250,7 +250,7 @@ func decodeRuntimeHistoryValue(keyType string, value []byte) RuntimeHistoryValue case "block": var block cborRuntimeHistoryAnnotatedBlock if err := cbor.Unmarshal(value, &block); err != nil { - info.BlockError = err.Error() + info.RawError = err.Error() return info } blockInfo := &RuntimeHistoryBlockInfo{ @@ -263,14 +263,14 @@ func decodeRuntimeHistoryValue(keyType string, value []byte) RuntimeHistoryValue blockInfo.RuntimeHeight = h.Round blockInfo.Timestamp = time.Unix(int64(h.Timestamp), 0).UTC().Format(time.RFC3339) blockInfo.HeaderType = headerTypeName(h.HeaderType) - blockInfo.StateRoot = truncateHex(h.StateRoot, TruncateHashSize) + blockInfo.StateRoot = formatRawValue(h.StateRoot, TruncateHashLen) } info.Block = blockInfo case "round_results": var results cborRuntimeHistoryRoundResults if err := cbor.Unmarshal(value, &results); err != nil { - info.RoundResultsError = err.Error() + info.RawError = err.Error() return info } info.RoundResults = &RuntimeHistoryRoundResultsInfo{ @@ -304,8 +304,11 @@ func headerTypeName(headerType uint8) string { // decodeRuntimeLeafValue decodes MKVS leaf value and returns structured info with valueType classification. // See: _oasis-core/go/runtime/transaction/transaction.go:129-150 (artifacts) func decodeRuntimeLeafValue(module string, key []byte, value []byte) *RuntimeLeafValueInfo { - info := &RuntimeLeafValueInfo{} - info.ValueType = "binary" // default + info := &RuntimeLeafValueInfo{ + ValueDump: formatRawValue(value, TruncateLongLen), + ValueSize: len(value), + ValueType: "unknown", // default + } // Check for EVM module data if len(module) >= 3 && module[:3] == "evm" { @@ -335,12 +338,12 @@ func decodeRuntimeLeafValue(module string, key []byte, value []byte) *RuntimeLea kind := key[33] if kind == 1 { // Input artifact - info.EVMTxInput, info.EVMTxInputError = decodeEVMTxInput(key, value) + info.EVMTxInput = decodeEVMTxInput(key, value) info.ValueType = "io_input" return info } else if kind == 2 { // Output artifact - info.EVMTxOutput, info.EVMTxOutputError = decodeEVMTxOutput(value) + info.EVMTxOutput = decodeEVMTxOutput(value) info.ValueType = "io_output" return info } @@ -351,10 +354,18 @@ func decodeRuntimeLeafValue(module string, key []byte, value []byte) *RuntimeLea if len(module) > 8 && module[:8] == "io_event" { // Special handling for EVM events if len(module) >= 12 && module[9:12] == "evm" { - info.EVMEvent, info.EVMEventError = decodeEVMEvent(value) + info.EVMEvent = decodeEVMEvent(value) info.ValueType = "evm_event" return info } + // Special handling for consensus_accounts events + if len(module) >= 25 && module[9:25] == "consensus_accounts" { + if evt := decodeRuntimeConsensusEvent(value); evt != nil { + info.RuntimeConsensusEvent = evt + info.ValueType = "runtime_consensus_accounts_event" + return info + } + } // Generic event decoding for non-EVM events var decoded interface{} @@ -363,19 +374,122 @@ func decodeRuntimeLeafValue(module string, key []byte, value []byte) *RuntimeLea info.CBOR = formatCBORDetailed(decoded) info.ValueType = "io_event" } else { - info.CBORError = err.Error() + info.ValueError = err.Error() } return info } + // Detect raw binary data based on module + switch module { + case "contracts:code": + // WASM bytecode - starts with magic bytes 0xFF060000734e615070 ("sNaPpY" Snappy compression) + if len(value) > 10 && value[0] == 0xFF { + info.ValueType = "wasm_bytecode" + info.CBOR = fmt.Sprintf("snappy_compressed_wasm (%d bytes)", len(value)) + return info + } + case "contracts:instance_state": + // Check if it's raw hash/encrypted data (typically 32-33 bytes, non-CBOR) + if len(value) >= 32 && len(value) <= 128 { + // Try CBOR first, but if it fails with "extraneous data" it's likely raw binary + var decoded interface{} + if err := cbor.Unmarshal(value, &decoded); err != nil { + // Check for characteristic CBOR errors indicating raw binary + if strings.Contains(err.Error(), "extraneous data") || + strings.Contains(err.Error(), "unexpected \"break\"") || + strings.Contains(err.Error(), "exceeded max number of elements") { + info.ValueType = "raw_binary" + info.CBOR = fmt.Sprintf("raw_state_data (%d bytes)", len(value)) + return info + } + info.ValueError = err.Error() + return info + } + // CBOR decode succeeded + info.CBOR = formatCBOR(decoded, len(value)) + info.ValueType = "cbor" + return info + } + } + // Try CBOR decode for regular state keys var decoded interface{} if err := cbor.Unmarshal(value, &decoded); err == nil { info.CBOR = formatCBOR(decoded, len(value)) info.ValueType = "cbor" } else { - info.CBORError = err.Error() + info.ValueError = err.Error() } return info } + +// convertRuntimeConsensusError converts CBOR error to output error. +func convertRuntimeConsensusError(err *cborRuntimeConsensusError) *RuntimeConsensusEventError { + if err == nil { + return nil + } + return &RuntimeConsensusEventError{Module: err.Module, Code: err.Code} +} + +// decodeRuntimeConsensusEvent decodes consensus_accounts events. +// See: _oasis-sdk/runtime-sdk/src/modules/consensus_accounts/mod.rs:106-157 +func decodeRuntimeConsensusEvent(value []byte) *RuntimeConsensusEventInfo { + var eventArray []interface{} + if err := cbor.Unmarshal(value, &eventArray); err != nil || len(eventArray) < 2 { + return nil + } + eventCode, ok := eventArray[0].(uint64) + if !ok { + return nil + } + + info := &RuntimeConsensusEventInfo{} + switch eventCode { + case 1, 2, 3: // Deposit, Withdraw, Delegate + var event cborRuntimeConsensusTransferEvent + if err := cbor.Unmarshal(value, &event); err != nil { + return nil + } + if eventCode == 1 { + info.EventType = "deposit" + } else if eventCode == 2 { + info.EventType = "withdraw" + } else { + info.EventType = "delegate" + } + info.From = event.From.String() + info.Nonce = event.Nonce + info.To = event.To.String() + info.Amount = event.Amount.String() + info.Error = convertRuntimeConsensusError(event.Error) + + case 4: // UndelegateStart + var event cborRuntimeConsensusUndelegateStartEvent + if err := cbor.Unmarshal(value, &event); err != nil { + return nil + } + info.EventType = "undelegate_start" + info.From = event.From.String() + info.Nonce = event.Nonce + info.To = event.To.String() + info.Shares = new(big.Int).SetBytes(event.Shares).String() + info.DebondEndTime = event.DebondEndTime + info.Error = convertRuntimeConsensusError(event.Error) + + case 5: // UndelegateDone + var event cborRuntimeConsensusUndelegateDoneEvent + if err := cbor.Unmarshal(value, &event); err != nil { + return nil + } + info.EventType = "undelegate_done" + info.From = event.From.String() + info.To = event.To.String() + info.Shares = new(big.Int).SetBytes(event.Shares).String() + info.Amount = event.Amount.String() + + default: + return nil + } + return info +} diff --git a/badgerdb-sampler/main.go b/badgerdb-sampler/main.go index a93529f..50a57e2 100644 --- a/badgerdb-sampler/main.go +++ b/badgerdb-sampler/main.go @@ -16,15 +16,9 @@ var BadgerVersion string // Sample contains both raw and decoded key/value information type Sample struct { - KeyRaw string `json:"key_raw"` - KeySize int `json:"key_size"` KeyType string `json:"key_type"` - Key interface{} `json:"key"` // Decoded key (renamed from Key) - KeyError string `json:"key_error,omitempty"` // Error decoding key - ValueRaw string `json:"value_raw"` - ValueSize int `json:"value_size"` - Value interface{} `json:"value"` // Decoded value (renamed from Value) - ValueError string `json:"value_error,omitempty"` // Error decoding value + Key interface{} `json:"key"` + Value interface{} `json:"value"` Timestamp int64 `json:"timestamp,omitempty"` } @@ -33,9 +27,10 @@ type DBStats struct { DatabasePath string `json:"database_path"` DatabaseType string `json:"database_type"` BadgerDBVersion string `json:"badgerdb_version"` - DatabaseSize int64 `json:"database_size_bytes,omitempty"` + DatabaseSize int64 `json:"database_size_bytes"` SampleCount int `json:"sample_count"` - KeyTypeCounts map[string]int `json:"key_type_counts,omitempty"` + KeyTypeCounts map[string]int `json:"key_type_counts"` + ErrorCounts map[string]int `json:"error_counts"` Samples []Sample `json:"samples"` } @@ -84,6 +79,7 @@ func main() { BadgerDBVersion: BadgerVersion, DatabaseSize: dbSize, KeyTypeCounts: make(map[string]int), + ErrorCounts: make(map[string]int), Samples: []Sample{}, } @@ -94,6 +90,22 @@ func main() { log.Fatalf("Error collecting samples: %v", err) } + // Extract and count errors from all samples + fmt.Fprintf(os.Stderr, "Extracting errors from samples...\n") + for _, sample := range stats.Samples { + // Extract errors from key + keyErrors := extractErrors(sample.Key, "key", 10) + for _, errMsg := range keyErrors { + stats.ErrorCounts[errMsg]++ + } + + // Extract errors from value + valueErrors := extractErrors(sample.Value, "value", 10) + for _, errMsg := range valueErrors { + stats.ErrorCounts[errMsg]++ + } + } + // Print results fmt.Printf("Results:\n") output, err := json.MarshalIndent(stats, "", " ") @@ -174,69 +186,82 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { key := item.Key() sample := Sample{ - KeyRaw: truncateHex(key, TruncateLongSize), - KeySize: len(key), - KeyType: "unknown", - Key: nil, + KeyType: "unknown", } // Decode key based on database type switch stats.DatabaseType { case "consensus-blockstore": keyInfo := decodeConsensusBlockstoreKey(key) - sample.KeyType = keyInfo.KeyType sample.Key = keyInfo + sample.KeyType = keyInfo.KeyType case "consensus-evidence": keyInfo := decodeConsensusEvidenceKey(key) - sample.KeyType = keyInfo.KeyType sample.Key = keyInfo + sample.KeyType = keyInfo.KeyType case "consensus-mkvs": keyInfo := decodeConsensusMkvsKey(key) - sample.KeyType = keyInfo.KeyType sample.Key = keyInfo + sample.KeyType = keyInfo.KeyType case "consensus-state": keyInfo := decodeConsensusStateKey(key) - sample.KeyType = keyInfo.KeyType sample.Key = keyInfo + sample.KeyType = keyInfo.KeyType case "runtime-mkvs": keyInfo := decodeRuntimeMkvsKey(key) - sample.KeyType = keyInfo.KeyType sample.Key = keyInfo + sample.KeyType = keyInfo.KeyType case "runtime-history": keyInfo := decodeRuntimeHistoryKey(key) - sample.KeyType = keyInfo.KeyType sample.Key = keyInfo + sample.KeyType = keyInfo.KeyType } // Fetch and decode value - err := item.Value(func(val []byte) error { - sample.ValueSize = len(val) - sample.ValueRaw = truncateHex(val, TruncateLongSize) - sample.Value = nil - - // Decode value based on database type - switch stats.DatabaseType { - case "consensus-blockstore": - valueInfo := decodeConsensusBlockstoreValue(sample.KeyType, val) - sample.Value = valueInfo - sample.Timestamp = valueInfo.Timestamp - case "consensus-evidence": - value, err := decodeConsensusEvidenceValue(sample.KeyType, val) - sample.Value = value - sample.ValueError = err - case "consensus-mkvs": + switch stats.DatabaseType { + case "consensus-blockstore": + val, err := item.ValueCopy(nil) + if err != nil { + sample.Value = &ConsensusBlockstoreValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err)} + } else { + sample.Value = decodeConsensusBlockstoreValue(sample.KeyType, val) + sample.Timestamp = sample.Value.(*ConsensusBlockstoreValueInfo).Timestamp + } + case "consensus-evidence": + val, err := item.ValueCopy(nil) + if err != nil { + sample.Value = &ConsensusEvidenceValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err)} + } else { + sample.Value = decodeConsensusEvidenceValue(sample.KeyType, val) + } + case "consensus-mkvs": + val, err := item.ValueCopy(nil) + if err != nil { + sample.Value = &ConsensusMkvsValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err)} + } else { sample.Value = decodeConsensusMkvsValue(sample.KeyType, val) - case "consensus-state": + } + case "consensus-state": + val, err := item.ValueCopy(nil) + if err != nil { + sample.Value = &ConsensusStateValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err)} + } else { sample.Value = decodeConsensusStateValue(sample.KeyType, val) - case "runtime-mkvs": + } + case "runtime-mkvs": + val, err := item.ValueCopy(nil) + if err != nil { + sample.Value = &RuntimeMkvsValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err)} + } else { sample.Value = decodeRuntimeMkvsValue(sample.KeyType, val) - case "runtime-history": + } + case "runtime-history": + val, err := item.ValueCopy(nil) + if err != nil { + sample.Value = &RuntimeHistoryValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err)} + } else { sample.Value = decodeRuntimeHistoryValue(sample.KeyType, val) } - return nil - }) - if err != nil { - log.Printf("Error reading value for key %s: %v", sample.KeyRaw, err) } stats.Samples = append(stats.Samples, sample) diff --git a/badgerdb-sampler/types_consensus.go b/badgerdb-sampler/types_consensus.go index 5f00803..8df6b1a 100644 --- a/badgerdb-sampler/types_consensus.go +++ b/badgerdb-sampler/types_consensus.go @@ -18,28 +18,34 @@ import ( // ConsensusBlockstoreKeyInfo represents a decoded consensus-blockstore key. // See: tendermint/store/store.go for key formats (H:, P:, C:, SC:, BH:) type ConsensusBlockstoreKeyInfo struct { - KeyType string `json:"key_type"` // "blockstore_state", "block_meta", "block_part", "block_commit", "seen_commit", "block_hash", "unknown" - ConsensusHeight int64 `json:"consensus_height,omitempty"` // For H:, C:, SC:, P: keys - PartIndex int `json:"part_index,omitempty"` // For P: keys - Hash string `json:"hash,omitempty"` // For BH: keys - KeySize int `json:"key_size"` - KeyRaw string `json:"key_raw,omitempty"` // Original ASCII key + // Raw fields + KeyDump string `json:"key_dump,omitempty"` + KeySize int `json:"key_size"` + KeyError string `json:"key_error,omitempty"` + + // Decoded fields + KeyType string `json:"key_type"` // "blockstore_state", "block_meta", "block_part", "block_commit", "seen_commit", "block_hash", "unknown" + ConsensusHeight int64 `json:"consensus_height,omitempty"` // For H:, C:, SC:, P: keys + PartIndex int `json:"part_index,omitempty"` // For P: keys + Hash string `json:"hash,omitempty"` // For BH: keys } // ConsensusBlockstoreValueInfo represents a decoded consensus-blockstore value. // See: tendermint/proto/tendermint/store/types.proto (BlockStoreState) // See: tendermint/proto/tendermint/types/types.proto (BlockMeta, Part, Commit) type ConsensusBlockstoreValueInfo struct { - Timestamp int64 `json:"timestamp,omitempty"` // Unix timestamp from block meta - State *ConsensusBlockStoreState `json:"state,omitempty"` - StateError string `json:"state_error,omitempty"` // Error decoding state - BlockMeta *ConsensusBlockMetaInfo `json:"block_meta,omitempty"` - BlockMetaError string `json:"block_meta_error,omitempty"` // Error decoding block meta - Part *ConsensusPartInfo `json:"part,omitempty"` - PartError string `json:"part_error,omitempty"` // Error decoding part - Commit *ConsensusCommitInfo `json:"commit,omitempty"` - CommitError string `json:"commit_error,omitempty"` // Error decoding commit - HashHeight int64 `json:"hash_height,omitempty"` // For block_hash type + // Raw fields + RawDump string `json:"raw_dump,omitempty"` + RawSize int `json:"raw_size"` + RawError string `json:"raw_error,omitempty"` + + // Decoded fields + Timestamp int64 `json:"timestamp,omitempty"` // Unix timestamp from block meta + State *ConsensusBlockStoreState `json:"state,omitempty"` + BlockMeta *ConsensusBlockMetaInfo `json:"block_meta,omitempty"` + Part *ConsensusPartInfo `json:"part,omitempty"` + Commit *ConsensusCommitInfo `json:"commit,omitempty"` + HashHeight int64 `json:"hash_height,omitempty"` // For block_hash type } // ConsensusBlockStoreState represents BlockStoreState from tendermint. @@ -82,23 +88,31 @@ type ConsensusCommitInfo struct { // ConsensusEvidenceKeyInfo represents a decoded consensus-evidence key. // See: tendermint/store/evidence/pool.go for key formats type ConsensusEvidenceKeyInfo struct { + // Raw fields + KeyDump string `json:"key_dump,omitempty"` + KeySize int `json:"key_size"` + KeyError string `json:"key_error,omitempty"` + + // Decoded fields KeyType string `json:"key_type"` PrefixByte byte `json:"prefix_byte,omitempty"` - KeySize int `json:"key_size"` - KeyHex string `json:"key_hex,omitempty"` // hex } // ConsensusEvidenceValueInfo represents a decoded consensus-evidence value. // See: tendermint/proto/tendermint/types/evidence.proto (DuplicateVoteEvidence) type ConsensusEvidenceValueInfo struct { - ValueSize int `json:"value_size"` - ValueHex string `json:"value_hex,omitempty"` // Truncated hex dump + // Raw fields + RawDump string `json:"raw_dump,omitempty"` + RawSize int `json:"raw_size"` + RawError string `json:"raw_error,omitempty"` + + // Decoded fields + EvidenceType string `json:"evidence_type,omitempty"` // "duplicate_vote", "light_client_attack", "unknown" VoteAHeight int64 `json:"vote_a_height,omitempty"` VoteBHeight int64 `json:"vote_b_height,omitempty"` - EvidenceType string `json:"evidence_type,omitempty"` // "duplicate_vote", "light_client_attack", "unknown" TotalVotingPower int64 `json:"total_voting_power,omitempty"` ValidatorPower int64 `json:"validator_power,omitempty"` - Timestamp string `json:"timestamp,omitempty"` // RFC3339 format + Timestamp string `json:"timestamp,omitempty"` // RFC3339 format } // ============================================================================= @@ -108,38 +122,53 @@ type ConsensusEvidenceValueInfo struct { // ConsensusMkvsKeyInfo represents a decoded consensus-mkvs key. // See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 type ConsensusMkvsKeyInfo struct { - KeySize int `json:"key_size"` - KeyHex string `json:"key_hex,omitempty"` // hex, truncated + // Raw fields + KeyDump string `json:"key_dump,omitempty"` + KeySize int `json:"key_size"` + KeyError string `json:"key_error,omitempty"` + + // Decoded fields KeyType string `json:"key_type"` // "node", "write_log", "roots_metadata", "root_updated_nodes", "metadata", "multipart_restore_log", "root_node", "unknown" - ConsensusHeight uint64 `json:"consensus_height,omitempty"` // For write_log, roots_metadata, root_updated_nodes + ConsensusHeight int64 `json:"consensus_height,omitempty"` // For write_log, roots_metadata, root_updated_nodes Hash string `json:"hash,omitempty"` // hex, truncated (partial key data) - RootType byte `json:"root_type,omitempty"` + RootType string `json:"root_type,omitempty"` } // ConsensusMkvsValueInfo represents a decoded consensus-mkvs value (node). // See: _oasis-core/go/storage/mkvs/node/node.go:26-32 (prefixes), 294-309 (InternalNode), 531-537 (LeafNode) type ConsensusMkvsValueInfo struct { - NodeType string `json:"node_type"` // "leaf", "internal", "nil", "non_node", "unknown" - NodeSize int `json:"node_size"` // Total MKVS node size - NodeHex string `json:"node_hex,omitempty"` // Raw node dump (hex, truncated) - Leaf *ConsensusMkvsLeafInfo `json:"leaf,omitempty"` - LeafError string `json:"leaf_error,omitempty"` // Error decoding leaf node - Internal *ConsensusMkvsInternalInfo `json:"internal,omitempty"` - InternalError string `json:"internal_error,omitempty"` // Error decoding internal node - NodeError string `json:"node_error,omitempty"` // Structural parsing error + // Raw fields + RawDump string `json:"raw_dump,omitempty"` + RawSize int `json:"raw_size"` + RawError string `json:"raw_error,omitempty"` + + // Node info + NodeType string `json:"node_type"` // "leaf", "internal", "nil", "non_node", "unknown" + Leaf *ConsensusMkvsLeafInfo `json:"leaf,omitempty"` + Internal *ConsensusMkvsInternalInfo `json:"internal,omitempty"` } // ConsensusMkvsLeafInfo represents a decoded MKVS LeafNode for consensus. // See: _oasis-core/go/storage/mkvs/node/node.go:531-537 (LeafNode) // See: _oasis-core/go/consensus/tendermint/apps/*/state/state.go (module prefixes) type ConsensusMkvsLeafInfo struct { - Module string `json:"module"` - KeySize int `json:"key_size"` - KeyHex string `json:"key_hex,omitempty"` // hex, truncated - OasisAddress string `json:"oasis_address,omitempty"` // bech32 oasis1... address if applicable - ValueSize int `json:"value_size"` - ValueHex string `json:"value_hex,omitempty"` // hex, truncated - ValueFormatted interface{} `json:"value_formatted,omitempty"` // CBOR formatted output from formatCBOR() + // Leaf key (extracted from MKVS node) + KeyDump string `json:"key_dump,omitempty"` + KeySize int `json:"key_size"` + + // Decoded key fields + Module string `json:"module"` + KeyType string `json:"key_type,omitempty"` + OasisAddress string `json:"oasis_address,omitempty"` // bech32 oasis1... address if applicable + + // Leaf value raw data + ValueDump string `json:"value_dump,omitempty"` + ValueSize int `json:"value_size"` + ValueError string `json:"value_error,omitempty"` + + // Decoded value + ValueType string `json:"value_type,omitempty"` + Value interface{} `json:"value,omitempty"` } // ConsensusMkvsInternalInfo represents a decoded MKVS InternalNode. @@ -158,26 +187,30 @@ type ConsensusMkvsInternalInfo struct { // ConsensusStateKeyInfo represents a decoded consensus-state key. // See: tendermint/state/store.go for key formats (abciResponsesKey, validatorsKey, etc.) type ConsensusStateKeyInfo struct { - KeyType string `json:"key_type"` // "abci_responses", "consensus_params", "validators", "state", "genesis", "text_key", "binary", "unknown" - KeySize int `json:"key_size"` - KeyHex string `json:"key_hex,omitempty"` // hex, truncated - ConsensusHeight int64 `json:"consensus_height,omitempty"` // For abci_responses (extracted from key) + // Raw fields + KeyDump string `json:"key_dump,omitempty"` + KeySize int `json:"key_size"` + KeyError string `json:"key_error,omitempty"` + + // Decoded fields + KeyType string `json:"key_type"` // "abci_responses", "consensus_params", "validators", "state", "genesis", "text_key", "binary", "unknown" + ConsensusHeight int64 `json:"consensus_height,omitempty"` // For abci_responses (extracted from key) IsBinary bool `json:"is_binary,omitempty"` } // ConsensusStateValueInfo represents a decoded consensus-state value. // See: tendermint/proto/tendermint/state/types.proto (various state types) type ConsensusStateValueInfo struct { - ValueSize int `json:"value_size"` - ValueHex string `json:"value_hex,omitempty"` // Raw hex dump - ABCIResponse *ConsensusABCIResponseInfo `json:"abci_response,omitempty"` // For abci_responses - ABCIResponseError string `json:"abci_response_error,omitempty"` // Error decoding ABCI responses - ConsensusParams *tmConsensusParams `json:"consensus_params,omitempty"` // For consensus_params - ConsensusParamsError string `json:"consensus_params_error,omitempty"` // Error decoding consensus params - ConsensusValidators *tmValidatorSet `json:"consensus_validators,omitempty"` // For validators - ValidatorsError string `json:"validators_error,omitempty"` // Error decoding validators - ConsensusState *tmState `json:"consensus_state,omitempty"` // For state - StateError string `json:"state_error,omitempty"` // Error decoding state + // Raw fields + RawDump string `json:"raw_dump,omitempty"` + RawSize int `json:"raw_size"` + RawError string `json:"raw_error,omitempty"` + + // Decoded content - only ONE populated + ABCIResponse *ConsensusABCIResponseInfo `json:"abci_response,omitempty"` // For abci_responses + ConsensusParams *tmConsensusParams `json:"consensus_params,omitempty"` // For consensus_params + ConsensusValidators *tmValidatorSet `json:"consensus_validators,omitempty"` // For validators + ConsensusState *tmState `json:"consensus_state,omitempty"` // For state } // ConsensusABCIResponseInfo represents decoded ABCI response summary. @@ -285,10 +318,10 @@ type cborConsensusSignedTransaction struct { // cborConsensusInnerTransaction represents an unsigned consensus transaction. // See: _oasis-core/go/consensus/api/transaction/transaction.go:43-54 type cborConsensusInnerTransaction struct { - Nonce uint64 - Fee *cborConsensusFee - Method string - Body cbor.RawMessage + Nonce uint64 `json:"nonce"` + Fee *cborConsensusFee `json:"fee"` + Method string `json:"method"` + Body cbor.RawMessage `json:"body"` } // cborConsensusFee represents transaction fee. diff --git a/badgerdb-sampler/types_evm.go b/badgerdb-sampler/types_evm.go index 78cc857..b19e0f1 100644 --- a/badgerdb-sampler/types_evm.go +++ b/badgerdb-sampler/types_evm.go @@ -17,6 +17,12 @@ type EVMDataInfo struct { // See: _oasis-sdk/runtime-sdk/modules/evm/src/lib.rs:253-263 (Event::Log) // See: _oasis-sdk/client-sdk/go/modules/evm/types.go:56-61 (Event) type EVMEventInfo struct { + // Raw fields + RawDump string `json:"raw_dump,omitempty"` + RawSize int `json:"raw_size"` + RawError string `json:"raw_error,omitempty"` + + // Decoded fields Address string `json:"address"` // H160 contract address (0x-prefixed hex) TopicCount int `json:"topic_count"` // Number of topics (0-4) Topics []string `json:"topics,omitempty"` // H256 topics array (0x-prefixed hex) @@ -29,31 +35,48 @@ type EVMEventInfo struct { // EVMTxInputInfo represents decoded EVM transaction input artifacts. // See: _oasis-core/go/runtime/transaction/transaction.go:126-140 (inputArtifacts) type EVMTxInputInfo struct { - TxHash string `json:"tx_hash"` // Transaction hash (0x-prefixed hex) - BatchOrder uint32 `json:"batch_order"` // Order within batch - TxInputSize int `json:"tx_input_size"` // Raw input size - TxInputHex string `json:"tx_input_hex,omitempty"` // Truncated raw input (hex) - Method string `json:"method,omitempty"` // SDK method (e.g., "evm.Call") - EVMCall *EVMCallInfo `json:"evm_call,omitempty"` // EVM-specific call details + // Raw fields + RawDump string `json:"raw_dump,omitempty"` + RawSize int `json:"raw_size"` + RawError string `json:"raw_error,omitempty"` + + // Decoded fields + TxHash string `json:"tx_hash"` // Transaction hash (0x-prefixed hex) + BatchOrder uint32 `json:"batch_order"` // Order within batch + Method string `json:"method,omitempty"` // SDK method (e.g., "evm.Call") + EVMTx *EVMTransactionInfo `json:"evm_tx,omitempty"` // Decoded EVM transaction } -// EVMCallInfo represents EVM-specific transaction call details. -// See: _oasis-sdk/runtime-sdk/modules/evm/src/types.rs:10-16 (Call/Create) -type EVMCallInfo struct { - Type string `json:"type"` // "call" or "create" - Address string `json:"address,omitempty"` // H160 target address (0x-prefixed hex) - Value string `json:"value,omitempty"` // U256 value in wei (decimal string) - DataSize int `json:"data_size"` // Size of call data or init code - DataHex string `json:"data_hex,omitempty"` // Truncated raw data (0x-prefixed hex) +// EVMTransactionInfo represents a decoded EVM transaction. +type EVMTransactionInfo struct { + // Raw EVM transaction body bytes (CBOR) + RawDump string `json:"raw_dump,omitempty"` + RawSize int `json:"raw_size"` + RawError string `json:"raw_error,omitempty"` + + // Decoded EVM transaction fields + Type string `json:"type"` // "call" or "create" + From string `json:"from,omitempty"` // 0x prefix + To string `json:"to,omitempty"` // 0x prefix + Value string `json:"value,omitempty"` // Wei as decimal string + GasLimit uint64 `json:"gas_limit,omitempty"` + GasPrice string `json:"gas_price,omitempty"` // Wei as decimal string + Nonce uint64 `json:"nonce,omitempty"` + DataSize int `json:"data_size"` + DataDump string `json:"data_dump,omitempty"` // 0x-hex } // EVMTxOutputInfo represents decoded EVM transaction output artifacts. // See: _oasis-core/go/runtime/transaction/transaction.go:142-150 (outputArtifacts) type EVMTxOutputInfo struct { - TxOutputSize int `json:"tx_output_size"` // Raw output size - TxOutputHex string `json:"tx_output_hex,omitempty"` // Truncated raw output (hex) - Success bool `json:"success"` // Transaction succeeded - ResultSize int `json:"result_size,omitempty"` // Size of result data - ResultHex string `json:"result_hex,omitempty"` // Truncated raw result (0x-prefixed hex) - Error string `json:"error,omitempty"` // Execution error message if failed (kept in child) + // Raw fields + RawDump string `json:"raw_dump,omitempty"` + RawSize int `json:"raw_size"` + RawError string `json:"raw_error,omitempty"` + + // Decoded execution status fields + SuccessExecution bool `json:"success"` + ResultSize int `json:"result_size,omitempty"` + ResultDump string `json:"result_dump,omitempty"` + ErrorExecution string `json:"error,omitempty"` // Execution error (application data, not decoding error) } diff --git a/badgerdb-sampler/types_runtime.go b/badgerdb-sampler/types_runtime.go index 307497b..72880cb 100644 --- a/badgerdb-sampler/types_runtime.go +++ b/badgerdb-sampler/types_runtime.go @@ -1,6 +1,10 @@ package main -import "github.com/fxamacker/cbor/v2" +import ( + "math/big" + + "github.com/fxamacker/cbor/v2" +) // ============================================================================= // Decoded Output Types (structured representations for JSON output) @@ -9,34 +13,44 @@ import "github.com/fxamacker/cbor/v2" // RuntimeMkvsKeyInfo represents a decoded runtime-mkvs key. // See: _oasis-core/go/storage/mkvs/db/badger/badger.go:31-66 type RuntimeMkvsKeyInfo struct { + // Raw fields + KeyDump string `json:"key_dump,omitempty"` + KeySize int `json:"key_size"` + KeyError string `json:"key_error,omitempty"` + + // Decoded fields KeyType string `json:"key_type"` // "node", "write_log", "roots_metadata", "root_updated_nodes", "metadata", "unknown" RuntimeHeight uint64 `json:"runtime_height,omitempty"` // For write_log, roots_metadata, root_updated_nodes Hash string `json:"hash,omitempty"` // For node (hex, truncated) } -// RuntimeMkvsNodeInfo represents a decoded runtime-mkvs value (node). +// RuntimeMkvsValueInfo represents a decoded runtime-mkvs value (node). // See: _oasis-core/go/storage/mkvs/node/node.go:26-32 (prefixes), 294-309 (InternalNode), 531-537 (LeafNode) -type RuntimeMkvsNodeInfo struct { - NodeType string `json:"node_type"` // "leaf", "internal", "nil", "non_node", "unknown" - NodeSize int `json:"node_size"` // Total MKVS node size (renamed from value_size for clarity) - NodeHex string `json:"node_hex,omitempty"` // Raw node dump (hex, truncated) - Leaf *RuntimeMkvsLeafInfo `json:"leaf,omitempty"` - LeafError string `json:"leaf_error,omitempty"` // Error decoding leaf node - Internal *RuntimeMkvsInternalInfo `json:"internal,omitempty"` - InternalError string `json:"internal_error,omitempty"` // Error decoding internal node - NodeError string `json:"node_error,omitempty"` // Structural parsing error +type RuntimeMkvsValueInfo struct { + // Raw fields + RawDump string `json:"raw_dump,omitempty"` + RawSize int `json:"raw_size"` + RawError string `json:"raw_error,omitempty"` + + // Node info + NodeType string `json:"node_type"` // "leaf", "internal", "nil", "non_node", "unknown" + Leaf *RuntimeMkvsLeafInfo `json:"leaf,omitempty"` + Internal *RuntimeMkvsInternalInfo `json:"internal,omitempty"` } // RuntimeMkvsLeafInfo represents a decoded MKVS LeafNode. // See: _oasis-core/go/storage/mkvs/node/node.go:531-537 type RuntimeMkvsLeafInfo struct { - Module string `json:"module"` - KeySize int `json:"key_size"` - KeyHex string `json:"key_hex,omitempty"` // hex, truncated - ValueSize int `json:"value_size"` - ValueHex string `json:"value_hex,omitempty"` // hex, truncated - Value *RuntimeLeafValueInfo `json:"value,omitempty"` // Decoded value (renamed from ValueDecoded) - ValueError string `json:"value_error,omitempty"` // Error decoding value + // Leaf key (extracted from MKVS node) + KeyDump string `json:"key_dump,omitempty"` + KeySize int `json:"key_size"` + + // Decoded key fields + Module string `json:"module"` + KeyType string `json:"key_type,omitempty"` + + // Nested value (has its own raw representation) + Value *RuntimeLeafValueInfo `json:"value,omitempty"` } // RuntimeMkvsInternalInfo represents a decoded MKVS InternalNode. @@ -51,19 +65,28 @@ type RuntimeMkvsInternalInfo struct { // RuntimeHistoryKeyInfo represents a decoded runtime-history key. // See: _oasis-core/go/runtime/history/db.go:19-31 type RuntimeHistoryKeyInfo struct { - KeyType string `json:"key_type"` // "metadata", "block", "round_results", "unknown" - RuntimeHeight uint64 `json:"runtime_height,omitempty"` // For block, round_results (runtime block height) - ExtraData string `json:"extra,omitempty"` // Unexpected extra bytes (hex) + // Raw fields + KeyDump string `json:"key_dump,omitempty"` + KeySize int `json:"key_size"` + KeyError string `json:"key_error,omitempty"` + + // Decoded fields + KeyType string `json:"key_type"` // "metadata", "block", "round_results", "unknown" + RuntimeHeight uint64 `json:"runtime_height,omitempty"` // For block, round_results (runtime block height) + ExtraData string `json:"extra,omitempty"` // Unexpected extra bytes (hex) } // RuntimeHistoryValueInfo represents a decoded runtime-history value. type RuntimeHistoryValueInfo struct { - Metadata *RuntimeHistoryMetadataInfo `json:"metadata,omitempty"` - MetadataError string `json:"metadata_error,omitempty"` // Error decoding metadata - Block *RuntimeHistoryBlockInfo `json:"block,omitempty"` - BlockError string `json:"block_error,omitempty"` // Error decoding block - RoundResults *RuntimeHistoryRoundResultsInfo `json:"round_results,omitempty"` - RoundResultsError string `json:"round_results_error,omitempty"` // Error decoding round results + // Raw fields + RawDump string `json:"raw_dump,omitempty"` + RawSize int `json:"raw_size"` + RawError string `json:"raw_error,omitempty"` + + // Decoded content - only ONE populated + Metadata *RuntimeHistoryMetadataInfo `json:"metadata,omitempty"` + Block *RuntimeHistoryBlockInfo `json:"block,omitempty"` + RoundResults *RuntimeHistoryRoundResultsInfo `json:"round_results,omitempty"` } // RuntimeHistoryMetadataInfo represents decoded runtime history metadata. @@ -97,19 +120,42 @@ type RuntimeHistoryRoundResultsInfo struct { // RuntimeLeafValueInfo represents a decoded MKVS leaf node value. // See: _oasis-core/go/runtime/transaction/transaction.go:129-150 (artifacts) -// Note: ValueHex and ValueSize are stored in parent RuntimeMkvsLeafInfo to avoid duplication type RuntimeLeafValueInfo struct { - ValueType string `json:"value_type,omitempty"` // Computed classification - CBOR interface{} `json:"cbor,omitempty"` // For io_event and cbor types - decoded CBOR data - CBORError string `json:"cbor_error,omitempty"` // Error decoding CBOR - EVM *EVMDataInfo `json:"evm,omitempty"` // For EVM-specific storage data - EVMError string `json:"evm_error,omitempty"` // Error decoding EVM storage - EVMEvent *EVMEventInfo `json:"evm_event,omitempty"` // For EVM event data - EVMEventError string `json:"evm_event_error,omitempty"` // Error decoding EVM event - EVMTxInput *EVMTxInputInfo `json:"evm_tx_input,omitempty"` // For EVM transaction input artifacts - EVMTxInputError string `json:"evm_tx_input_error,omitempty"` // Error decoding EVM tx input - EVMTxOutput *EVMTxOutputInfo `json:"evm_tx_output,omitempty"` // For EVM transaction output artifacts - EVMTxOutputError string `json:"evm_tx_output_error,omitempty"` // Error decoding EVM tx output + // Raw leaf value bytes (use Value* prefix for leaf-specific fields) + ValueDump string `json:"value_dump,omitempty"` + ValueSize int `json:"value_size"` + ValueError string `json:"value_error,omitempty"` + + // Classification + ValueType string `json:"value_type,omitempty"` + + // Decoded content - only ONE populated + CBOR interface{} `json:"cbor,omitempty"` + EVM *EVMDataInfo `json:"evm,omitempty"` + EVMEvent *EVMEventInfo `json:"evm_event,omitempty"` + EVMTxInput *EVMTxInputInfo `json:"evm_tx_input,omitempty"` + EVMTxOutput *EVMTxOutputInfo `json:"evm_tx_output,omitempty"` + RuntimeConsensusEvent *RuntimeConsensusEventInfo `json:"runtime_consensus_accounts_event,omitempty"` +} + +// RuntimeConsensusEventError represents consensus error. +// See: _oasis-sdk/runtime-sdk/src/modules/consensus_accounts/types.rs:207-215 +type RuntimeConsensusEventError struct { + Module string `json:"module,omitempty"` + Code uint32 `json:"code,omitempty"` +} + +// RuntimeConsensusEventInfo is decoded consensus_accounts event. +// See: _oasis-sdk/runtime-sdk/src/modules/consensus_accounts/mod.rs:105-157 +type RuntimeConsensusEventInfo struct { + EventType string `json:"event_type"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` + Nonce uint64 `json:"nonce,omitempty"` + Amount string `json:"amount,omitempty"` + Shares string `json:"shares,omitempty"` + DebondEndTime uint64 `json:"debond_end_time,omitempty"` + Error *RuntimeConsensusEventError `json:"error,omitempty"` } // ============================================================================= @@ -205,3 +251,74 @@ type cborRuntimeCallMapFormat struct { Body cbor.RawMessage `cbor:"body"` ReadOnly bool `cbor:"ro,omitempty"` } + +// ============================================================================= +// Runtime Event CBOR Deserialization Types +// ============================================================================= + +// RuntimeBaseUnits represents token::BaseUnits encoded as CBOR array [amount_bytes, denomination_bytes]. +// See: _oasis-sdk/runtime-sdk/src/types/token.rs:90 +// pub struct BaseUnits(pub u128, pub Denomination); +type RuntimeBaseUnits struct { + _ struct{} `cbor:",toarray"` + Amount []byte // u128 as big-endian bytes + Denomination []byte // Denomination as bytes (empty for native token) +} + +func (b *RuntimeBaseUnits) String() string { + if len(b.Amount) == 0 { + return "0" + } + return new(big.Int).SetBytes(b.Amount).String() +} + +// RuntimeAddress represents a 21-byte runtime address with bech32 encoding. +// See: _oasis-sdk/runtime-sdk/src/types/address.rs:79-80 +// pub struct Address([u8; ADDRESS_SIZE]); // ADDRESS_SIZE = 21 +// CBOR: Encodes as ByteString +// Display: Bech32 with HRP "oasis" +type RuntimeAddress [21]byte + +func (a RuntimeAddress) String() string { + return bech32Encode("oasis", a[:]) +} + +// cborRuntimeConsensusError represents error details from consensus layer. +// See: _oasis-sdk/runtime-sdk/src/modules/consensus_accounts/types.rs:207-215 +type cborRuntimeConsensusError struct { + Module string `cbor:"module,omitempty"` + Code uint32 `cbor:"code,omitempty"` +} + +// Deposit/Withdraw/Delegate events (codes 1-3). +// See: _oasis-sdk/runtime-sdk/src/modules/consensus_accounts/mod.rs:109-137 +type cborRuntimeConsensusTransferEvent struct { + _ struct{} `cbor:",toarray"` + From RuntimeAddress + Nonce uint64 + To RuntimeAddress + Amount RuntimeBaseUnits + Error *cborRuntimeConsensusError `cbor:"error,omitempty"` +} + +// UndelegateStart event (code 4). +// See: _oasis-sdk/runtime-sdk/src/modules/consensus_accounts/mod.rs:139-148 +type cborRuntimeConsensusUndelegateStartEvent struct { + _ struct{} `cbor:",toarray"` + From RuntimeAddress + Nonce uint64 + To RuntimeAddress + Shares []byte + DebondEndTime uint64 + Error *cborRuntimeConsensusError `cbor:"error,omitempty"` +} + +// UndelegateDone event (code 5). +// See: _oasis-sdk/runtime-sdk/src/modules/consensus_accounts/mod.rs:150-156 +type cborRuntimeConsensusUndelegateDoneEvent struct { + _ struct{} `cbor:",toarray"` + From RuntimeAddress + To RuntimeAddress + Shares []byte + Amount RuntimeBaseUnits +} diff --git a/badgerdb-sampler/utils.go b/badgerdb-sampler/utils.go index a4856ff..07439f2 100644 --- a/badgerdb-sampler/utils.go +++ b/badgerdb-sampler/utils.go @@ -1,16 +1,19 @@ package main import ( + "encoding/hex" "fmt" "math/big" + "reflect" "strings" + "unicode" ) const ( - // TruncateHashSize is the default hex character limit for hash truncation - TruncateHashSize = 16 - // TruncateLongSize is the default hex character limit for long data truncation - TruncateLongSize = 1000 + // TruncateHashLen is the default hex character limit for hash truncation + TruncateHashLen = 16 + // TruncateLongLen is the default hex character limit for long data truncation + TruncateLongLen = 1000 ) // extractModuleName extracts module name from MKVS leaf key and returns module:subtype description @@ -36,9 +39,9 @@ func extractModuleName(key []byte) string { } else if kind == 2 { kindStr = "output" } - return fmt.Sprintf("io_tx:%s (hash=%s)", kindStr, truncateHex(key[1:33], TruncateHashSize)) + return fmt.Sprintf("io_tx:%s", kindStr) } - return fmt.Sprintf("io_tx (hash=%s)", truncateHex(key[1:], TruncateHashSize)) + return "io_tx" case 'E': // Event tag prefix (0x45) // Key format: 'E' + tag_key (variable, module name) + tx_hash (32 bytes) @@ -75,7 +78,7 @@ func extractModuleName(key []byte) string { subKey := key[end:] return describeModuleKey(moduleName, subKey) } - return truncateHex(key, TruncateHashSize) + return truncateHex(key, TruncateHashLen) } // describeModuleKey returns module name with sub-key type description @@ -237,6 +240,23 @@ func isPrintableASCII(s string) bool { return true } +// formatRawValue formats data as ASCII or 0x-prefixed hex with truncation +// Returns ASCII string if printable, otherwise 0x-prefixed hex +func formatRawValue(data []byte, maxLen int) string { + if isPrintableASCII(string(data)) { + if len(data) > maxLen { + return string(data[:maxLen]) + "..." + } + return string(data) + } + // Hex formatting with 0x prefix + hexStr := hex.EncodeToString(data) + if len(hexStr) > maxLen { + return "0x" + hexStr[:maxLen] + "..." + } + return "0x" + hexStr +} + // truncateHex converts bytes to hex and truncates if too long. func truncateHex(data []byte, maxLen int) string { hex := fmt.Sprintf("%x", data) @@ -343,3 +363,75 @@ func bech32CreateChecksum(hrp string, data []int) []int { return checksum } +// extractErrors recursively extracts all non-empty fields ending with "Error" from a value. +func extractErrors(v interface{}, prefix string, maxDepth int) []string { + if maxDepth <= 0 || v == nil { + return []string{} + } + + val := reflect.ValueOf(v) + if !val.IsValid() { + return []string{} + } + + // Dereference pointers + for val.Kind() == reflect.Ptr { + if val.IsNil() { + return []string{} + } + val = val.Elem() + } + + results := []string{} + switch val.Kind() { + case reflect.Struct: + typ := val.Type() + for i := 0; i < val.NumField(); i++ { + field, fieldType := val.Field(i), typ.Field(i) + if !field.CanInterface() { + continue + } + jsonName := getJSONFieldName(fieldType) + // Collect error if field ends with "Error" and is non-empty string + if strings.HasSuffix(fieldType.Name, "Error") && field.Kind() == reflect.String { + if errStr := field.String(); errStr != "" { + results = append(results, fmt.Sprintf("%s.%s: %s", prefix, jsonName, errStr)) + } + } + results = append(results, extractErrors(field.Interface(), prefix+"."+jsonName, maxDepth-1)...) + } + case reflect.Interface: + if !val.IsNil() { + results = extractErrors(val.Interface(), prefix, maxDepth-1) + } + case reflect.Slice, reflect.Array: + for i := 0; i < val.Len(); i++ { + if elem := val.Index(i); elem.CanInterface() { + results = append(results, extractErrors(elem.Interface(), prefix+"[]", maxDepth-1)...) + } + } + } + return results +} + +// getJSONFieldName extracts JSON field name from struct tag, falls back to snake_case +func getJSONFieldName(field reflect.StructField) string { + if tag := field.Tag.Get("json"); tag != "" { + if name := strings.Split(tag, ",")[0]; name != "" { + return name + } + } + return toSnakeCase(field.Name) +} + +// toSnakeCase converts PascalCase to snake_case +func toSnakeCase(s string) string { + var result strings.Builder + for i, r := range s { + if i > 0 && unicode.IsUpper(r) { + result.WriteRune('_') + } + result.WriteRune(unicode.ToLower(r)) + } + return result.String() +} From f69055d21ebcc8613709651d8a44e08453aca2e3 Mon Sep 17 00:00:00 2001 From: gw0 Date: Mon, 1 Dec 2025 13:23:22 +0100 Subject: [PATCH 18/22] badgerdb-sampler: Add exact decoding of MKVS leafs and add support for CometBFT-v0.37 types --- badgerdb-sampler/db_v2.go | 6 - badgerdb-sampler/db_v3.go | 6 - badgerdb-sampler/db_v4.go | 6 - badgerdb-sampler/decode_consensus.go | 416 ++++++++++++++++++++------- badgerdb-sampler/decode_evm.go | 97 ++++--- badgerdb-sampler/decode_runtime.go | 41 +++ badgerdb-sampler/main.go | 34 ++- badgerdb-sampler/types_consensus.go | 297 +++++++++++++++---- badgerdb-sampler/types_evm.go | 25 +- badgerdb-sampler/types_runtime.go | 66 +++++ 10 files changed, 748 insertions(+), 246 deletions(-) diff --git a/badgerdb-sampler/db_v2.go b/badgerdb-sampler/db_v2.go index 728d6db..7a4451b 100644 --- a/badgerdb-sampler/db_v2.go +++ b/badgerdb-sampler/db_v2.go @@ -54,8 +54,6 @@ func openDatabase(path string) (*DB, error) { opts.ReadOnly = false opts.SyncWrites = false opts.NumMemtables = 1 // Minimize memtable usage - opts.MaxTableSize = 1 << 20 // 1MB table size (minimal) - opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 opts.NumCompactors = 0 @@ -85,8 +83,6 @@ func openDatabase(path string) (*DB, error) { opts.ReadOnly = false opts.SyncWrites = false opts.NumMemtables = 1 // Minimize memtable usage - opts.MaxTableSize = 1 << 20 // 1MB memtable (minimal) - v2 uses MaxTableSize instead of MemTableSize - opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 opts.NumCompactors = 0 @@ -134,8 +130,6 @@ func openDatabase(path string) (*DB, error) { opts.ReadOnly = false opts.SyncWrites = false opts.NumMemtables = 1 // Minimize memtable usage - opts.MaxTableSize = 1 << 20 // 1MB table size (minimal) - opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 opts.NumCompactors = 0 diff --git a/badgerdb-sampler/db_v3.go b/badgerdb-sampler/db_v3.go index ea0f02e..edcf0e5 100644 --- a/badgerdb-sampler/db_v3.go +++ b/badgerdb-sampler/db_v3.go @@ -54,8 +54,6 @@ func openDatabase(path string) (*DB, error) { opts.ReadOnly = false opts.SyncWrites = false opts.NumMemtables = 1 // Minimize memtable usage - opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) - v3 uses MemTableSize instead of MaxTableSize - opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 opts.NumCompactors = 0 @@ -85,8 +83,6 @@ func openDatabase(path string) (*DB, error) { opts.ReadOnly = false opts.SyncWrites = false opts.NumMemtables = 1 // Minimize memtable usage - opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) - v3 uses MemTableSize instead of MaxTableSize - opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 opts.NumCompactors = 0 @@ -135,8 +131,6 @@ func openDatabase(path string) (*DB, error) { opts.ReadOnly = false opts.SyncWrites = false opts.NumMemtables = 1 // Minimize memtable usage - opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) - v3 uses MemTableSize instead of MaxTableSize - opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 opts.NumCompactors = 0 diff --git a/badgerdb-sampler/db_v4.go b/badgerdb-sampler/db_v4.go index 04394cd..57db30f 100644 --- a/badgerdb-sampler/db_v4.go +++ b/badgerdb-sampler/db_v4.go @@ -54,8 +54,6 @@ func openDatabase(path string) (*DB, error) { opts.ReadOnly = false opts.SyncWrites = false opts.NumMemtables = 1 // Minimize memtable usage - opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) - v4 uses MemTableSize like v3 - opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 opts.NumCompactors = 0 @@ -85,8 +83,6 @@ func openDatabase(path string) (*DB, error) { opts.ReadOnly = false opts.SyncWrites = false opts.NumMemtables = 1 // Minimize memtable usage - opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) - v4 uses MemTableSize like v3 - opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 opts.NumCompactors = 0 @@ -135,8 +131,6 @@ func openDatabase(path string) (*DB, error) { opts.ReadOnly = false opts.SyncWrites = false opts.NumMemtables = 1 // Minimize memtable usage - opts.MemTableSize = 1 << 20 // 1MB memtable (minimal) - v4 uses MemTableSize like v3 - opts.ValueThreshold = 1 << 17 // 128KB - must be less than max batch size (~157KB with 1MB memtable) opts.NumLevelZeroTables = 100 opts.NumLevelZeroTablesStall = 200 opts.NumCompactors = 0 diff --git a/badgerdb-sampler/decode_consensus.go b/badgerdb-sampler/decode_consensus.go index eb7a4c1..969ac55 100644 --- a/badgerdb-sampler/decode_consensus.go +++ b/badgerdb-sampler/decode_consensus.go @@ -172,6 +172,8 @@ func decodeConsensusBlockstoreValue(keyType string, value []byte) *ConsensusBloc } // decodeConsensusEvidenceKey parses consensus-evidence key and returns structured info. +// Key format: 0x01 (dbVersion) + 0x00/0x01 (committed/pending) + "HEIGHT_HEX/HASH_HEX" +// See: cometbft/evidence/pool.go (keyCommitted, keyPending functions) func decodeConsensusEvidenceKey(key []byte) *ConsensusEvidenceKeyInfo { info := &ConsensusEvidenceKeyInfo{ KeyDump: formatRawValue(key, TruncateLongLen), @@ -189,15 +191,42 @@ func decodeConsensusEvidenceKey(key []byte) *ConsensusEvidenceKeyInfo { return info } - // Evidence DB is typically empty or uses simple key patterns - info.KeyType = fmt.Sprintf("type_%02x", key[1]) - info.PrefixByte = key[1] + // Second byte is the evidence state prefix + prefixByte := key[1] + info.PrefixByte = prefixByte + + switch prefixByte { + case 0x00: + info.KeyType = "committed" + case 0x01: + info.KeyType = "pending" + default: + info.KeyType = fmt.Sprintf("type_%02x", prefixByte) + return info + } + + // Parse key suffix: "HEIGHT_HEX/HASH_HEX" + if len(key) > 2 { + keySuffix := string(key[2:]) + parts := strings.Split(keySuffix, "/") + if len(parts) == 2 { + // Parse height from hex string (big-endian padded) + if h, err := strconv.ParseInt(parts[0], 16, 64); err == nil { + info.ConsensusHeight = h + } + // Store evidence hash + info.Hash = parts[1] + } + } + return info } // decodeConsensusEvidenceValue decodes evidence value and returns structured info. -func decodeConsensusEvidenceValue(keyType string, value []byte) *ConsensusEvidenceValueInfo { - info := &ConsensusEvidenceValueInfo{ +// Committed evidence stores only Int64Value (height), pending evidence stores full Evidence protobuf. +// See: cometbft/evidence/pool.go (addPendingEvidence, markEvidenceAsCommitted) +func decodeConsensusEvidenceValue(keyType string, value []byte) (info *ConsensusEvidenceValueInfo) { + info = &ConsensusEvidenceValueInfo{ RawDump: formatRawValue(value, TruncateLongLen), RawSize: len(value), } @@ -213,9 +242,49 @@ func decodeConsensusEvidenceValue(keyType string, value []byte) *ConsensusEviden } }() - // Try to decode as DuplicateVoteEvidence first + // Committed evidence only stores the block height as Int64Value + if keyType == "committed" { + var heightValue tmInt64Value + if err := proto.Unmarshal(value, &heightValue); err == nil { + info.EvidenceType = "committed_marker" + info.CommittedHeight = heightValue.Value + return info + } else { + info.RawError = fmt.Sprintf("failed to decode committed evidence Int64Value: %v", err) + info.EvidenceType = "unknown" + return info + } + } + + // Pending evidence stores full evidence protobuf - try CometBFT DuplicateVoteEvidence first + cbSuccess := false + var cbDve cbDuplicateVoteEvidence + func() { + defer func() { + recover() // Silently catch panics from CometBFT unmarshal + }() + if err := proto.Unmarshal(value, &cbDve); err == nil && cbDve.VoteA != nil && cbDve.VoteB != nil { + cbSuccess = true + } + }() + + if cbSuccess { + info.SchemaVersion = "cb-v0.37" + info.EvidenceType = "duplicate_vote" + info.VoteAHeight = cbDve.VoteA.Height + info.VoteBHeight = cbDve.VoteB.Height + info.TotalVotingPower = cbDve.TotalVotingPower + info.ValidatorPower = cbDve.ValidatorPower + if cbDve.Timestamp.Unix() > 0 { + info.Timestamp = cbDve.Timestamp.Format(time.RFC3339) + } + return info + } + + // Fall back to Tendermint v0.34 DuplicateVoteEvidence var dve tmDuplicateVoteEvidence if err := proto.Unmarshal(value, &dve); err == nil && dve.VoteA != nil { + info.SchemaVersion = "tm-v0.34" info.EvidenceType = "duplicate_vote" info.VoteAHeight = dve.VoteA.Height if dve.VoteB != nil { @@ -229,7 +298,7 @@ func decodeConsensusEvidenceValue(keyType string, value []byte) *ConsensusEviden return info } - // Try to decode as LightClientAttackEvidence + // Try to decode as LightClientAttackEvidence (same for both versions) var lca tmLightClientAttackEvidence if err := proto.Unmarshal(value, &lca); err == nil && lca.ConflictingBlock != nil { info.EvidenceType = "light_client_attack" @@ -243,7 +312,7 @@ func decodeConsensusEvidenceValue(keyType string, value []byte) *ConsensusEviden // If all parsing failed, show raw value info.EvidenceType = "unknown" if info.RawError == "" { - info.RawError = "unable to decode evidence format" + info.RawError = "failed to decode as any known evidence type" } return info } @@ -349,6 +418,14 @@ func decodeConsensusMkvsValue(keyType string, value []byte) *ConsensusMkvsValueI info.NodeType = "leaf" data := value[1:] + // Try pre-v21.1.0 format (skip 8-byte Version field) + if len(data) >= 10 { + testKeySize := int(binary.LittleEndian.Uint16(data[8:10])) + if testKeySize > 0 && testKeySize <= 10000 && len(data) >= 10+testKeySize+4 { + data = data[8:] + } + } + if len(data) < 2 { info.RawError = "key length missing" return info @@ -358,7 +435,7 @@ func decodeConsensusMkvsValue(keyType string, value []byte) *ConsensusMkvsValueI data = data[2:] if len(data) < keySize { - info.RawError = fmt.Sprintf("key truncated (expected %d bytes)", keySize) + info.RawError = fmt.Sprintf("key truncated") return info } @@ -401,7 +478,7 @@ func decodeConsensusMkvsValue(keyType string, value []byte) *ConsensusMkvsValueI leaf.ValueSize = valueSize if len(data) < valueSize { - info.RawError = fmt.Sprintf("value truncated (expected %d bytes)", valueSize) + info.RawError = fmt.Sprintf("value truncated") info.Leaf = leaf return info } @@ -409,13 +486,51 @@ func decodeConsensusMkvsValue(keyType string, value []byte) *ConsensusMkvsValueI leafValue := data[:valueSize] leaf.ValueDump = formatRawValue(leafValue, TruncateLongLen) - // Try CBOR decode - var decoded interface{} - if err := cbor.Unmarshal(leafValue, &decoded); err == nil { - leaf.Value = formatCBORDetailed(decoded) - leaf.ValueType = "cbor" + // Deterministic format lookup based on key prefix + format, exists := GetConsensusMKVSFormat(key) + if !exists { + // Unknown key prefix - try CBOR as fallback + var decoded interface{} + if err := cbor.Unmarshal(leafValue, &decoded); err == nil { + leaf.Value = formatCBORDetailed(decoded) + leaf.ValueType = "cbor" + } else { + leaf.ValueError = fmt.Sprintf("unknown key prefix 0x%02x: %s", key[0], err.Error()) + leaf.ValueType = "unknown" + } } else { - leaf.ValueError = err.Error() + // Known format - decode deterministically + switch format.Format { + case "cbor": + var decoded interface{} + if err := cbor.Unmarshal(leafValue, &decoded); err == nil { + leaf.Value = formatCBORDetailed(decoded) + leaf.ValueType = "cbor" + leaf.CBOR = format.Type // Store expected type + } else { + leaf.ValueError = err.Error() + leaf.ValueType = "cbor_error" + } + + case "binary": + // Raw binary data + leaf.ValueType = "raw_binary" + switch len(leafValue) { + case 32: + leaf.CBOR = fmt.Sprintf("%s (32-byte hash)", format.Description) + case 64: + leaf.CBOR = fmt.Sprintf("%s (64-byte signature)", format.Description) + case 65: + leaf.CBOR = fmt.Sprintf("%s (65-byte pubkey)", format.Description) + default: + leaf.CBOR = fmt.Sprintf("%s (%d bytes)", format.Description, len(leafValue)) + } + + case "empty": + // Index entry with empty value + leaf.ValueType = "empty" + leaf.CBOR = fmt.Sprintf("index entry: %s", format.Description) + } } info.Leaf = leaf @@ -425,6 +540,14 @@ func decodeConsensusMkvsValue(keyType string, value []byte) *ConsensusMkvsValueI info.NodeType = "internal" data := value[1:] + // Try pre-v21.1.0 format (skip 8-byte Version field) + if len(data) >= 10 { + testLabelBits := binary.LittleEndian.Uint16(data[8:10]) + if testLabelBits <= 2048 && len(data) >= 10+(int(testLabelBits)+7)/8+1 { + data = data[8:] + } + } + if len(data) < 2 { info.RawError = "label bits missing" return info @@ -480,7 +603,7 @@ func decodeConsensusMkvsValue(keyType string, value []byte) *ConsensusMkvsValueI } data = data[valueLen:] // skip value } else { - info.RawError = fmt.Sprintf("unexpected marker 0x%02x (expected 0x00 or 0x02)", data[0]) + info.RawError = fmt.Sprintf("unexpected marker 0x%02x", data[0]) info.Internal = internal return info } @@ -737,8 +860,8 @@ func decodeConsensusStateKey(key []byte) *ConsensusStateKeyInfo { } // decodeConsensusStateValue decodes state value and returns structured info. -func decodeConsensusStateValue(keyType string, value []byte) *ConsensusStateValueInfo { - info := &ConsensusStateValueInfo{ +func decodeConsensusStateValue(keyType string, value []byte) (info *ConsensusStateValueInfo) { + info = &ConsensusStateValueInfo{ RawDump: formatRawValue(value, TruncateLongLen), RawSize: len(value), } @@ -747,135 +870,224 @@ func decodeConsensusStateValue(keyType string, value []byte) *ConsensusStateValu return info } - // Add panic recovery for all protobuf unmarshaling operations + // Recover from protobuf panics that occur with schema mismatches defer func() { if r := recover(); r != nil { - // Protobuf panic occurred (likely schema mismatch) - if info.RawError == "" { - info.RawError = fmt.Sprintf("protobuf panic: %v", r) - } + info.RawError = fmt.Sprintf("protobuf panic: %v", r) } }() switch keyType { case "abci_responses": - // Try to decode ABCI responses (ResponseFinalizeBlock in newer Tendermint) - var resp tmABCIResponses - if err := proto.Unmarshal(value, &resp); err == nil { - // Validate that unmarshal actually decoded meaningful data - if resp.DeliverTxs == nil && resp.BeginBlock == nil && resp.EndBlock == nil { - info.RawError = "protobuf unmarshal succeeded but all fields are nil (schema mismatch)" - } else { - abciResponseInfo := &ConsensusABCIResponseInfo{} + // Try CometBFT FinalizeBlock format first + cbSuccess := false + var cbResp cbResponseFinalizeBlock + func() { + defer func() { + recover() // Silently catch panics from CometBFT unmarshal + }() + if err := proto.Unmarshal(value, &cbResp); err == nil && (len(cbResp.TxResults) > 0 || len(cbResp.Events) > 0 || len(cbResp.ValidatorUpdates) > 0) { + cbSuccess = true + } + }() - // Count deliver_tx results (transaction results) - if resp.DeliverTxs != nil { - abciResponseInfo.TxResultCount = len(resp.DeliverTxs) - } + if cbSuccess { + info.SchemaVersion = "cb-v0.37" + abciResponseInfo := &ConsensusABCIResponseInfo{} + + // Count tx results + if cbResp.TxResults != nil { + abciResponseInfo.TxResultCount = len(cbResp.TxResults) + } + + // Count validator updates + if cbResp.ValidatorUpdates != nil { + abciResponseInfo.ValidatorUpdates = len(cbResp.ValidatorUpdates) + } + + // Count event types from all sources + eventSummary := &ConsensusEventSummary{ + EventTypeCounts: make(map[string]int), + } + + // Decode transactions and collect events from TxResults + txSummary := &ConsensusTransactionSummary{ + MethodCounts: make(map[string]int), + } + + if cbResp.TxResults != nil { + for _, txResult := range cbResp.TxResults { + // Count and decode events + for _, event := range txResult.Events { + eventSummary.EventTypeCounts[event.Type]++ - // Count events from end_block - if resp.EndBlock != nil { - abciResponseInfo.EventCount = len(resp.EndBlock.Events) - if resp.EndBlock.ValidatorUpdates != nil { - abciResponseInfo.ValidatorUpdates = len(resp.EndBlock.ValidatorUpdates) + // Decode event if sample limit not reached + if decodedEvents, err := decodeConsensusEvent(event); err == nil { + for _, decodedEvent := range decodedEvents { + if len(eventSummary.Events) < 10 { + eventSummary.Events = append(eventSummary.Events, decodedEvent) + } + } + } } - } - // Count event types from all sources - eventSummary := &ConsensusEventSummary{ - EventTypeCounts: make(map[string]int), + // Decode transaction if present + if len(txResult.Data) > 0 { + if decodedTx, err := decodeConsensusTransaction(txResult.Data); err == nil { + txSummary.MethodCounts[decodedTx.Method]++ + // Only include first few transactions for sample + if len(txSummary.Transactions) < 10 { + txSummary.Transactions = append(txSummary.Transactions, decodedTx) + } + } + } } + } + + // Events from root Events field (BeginBlock-style events in CometBFT) + if cbResp.Events != nil { + for _, event := range cbResp.Events { + eventSummary.EventTypeCounts[event.Type]++ - // Decode transactions and collect events from DeliverTxs - txSummary := &ConsensusTransactionSummary{ - MethodCounts: make(map[string]int), + // Decode event if sample limit not reached + if decodedEvents, err := decodeConsensusEvent(event); err == nil { + for _, decodedEvent := range decodedEvents { + if len(eventSummary.Events) < 10 { + eventSummary.Events = append(eventSummary.Events, decodedEvent) + } + } + } } + } - if resp.DeliverTxs != nil { - for _, txResult := range resp.DeliverTxs { - // Count and decode events - for _, event := range txResult.Events { - eventSummary.EventTypeCounts[event.Type]++ + abciResponseInfo.EventCount = len(cbResp.Events) - // Decode event if sample limit not reached - if decodedEvents, err := decodeConsensusEvent(event); err == nil { - for _, decodedEvent := range decodedEvents { - if len(eventSummary.Events) < 10 { - eventSummary.Events = append(eventSummary.Events, decodedEvent) + if len(txSummary.MethodCounts) > 0 { + abciResponseInfo.TransactionSummary = txSummary + } + + if len(eventSummary.EventTypeCounts) > 0 { + abciResponseInfo.EventSummary = eventSummary + } + + info.ABCIResponse = abciResponseInfo + + } else { + // Fall back to Tendermint v0.34 format + var resp tmABCIResponses + if err := proto.Unmarshal(value, &resp); err == nil { + // Validate that unmarshal actually decoded meaningful data + if resp.DeliverTxs == nil && resp.BeginBlock == nil && resp.EndBlock == nil { + info.RawError = "protobuf unmarshal succeeded but all fields are nil (schema mismatch)" + } else { + info.SchemaVersion = "tm-v0.34" + abciResponseInfo := &ConsensusABCIResponseInfo{} + + // Count deliver_tx results (transaction results) + if resp.DeliverTxs != nil { + abciResponseInfo.TxResultCount = len(resp.DeliverTxs) + } + + // Count events from end_block + if resp.EndBlock != nil { + abciResponseInfo.EventCount = len(resp.EndBlock.Events) + if resp.EndBlock.ValidatorUpdates != nil { + abciResponseInfo.ValidatorUpdates = len(resp.EndBlock.ValidatorUpdates) + } + } + + // Count event types from all sources + eventSummary := &ConsensusEventSummary{ + EventTypeCounts: make(map[string]int), + } + + // Decode transactions and collect events from DeliverTxs + txSummary := &ConsensusTransactionSummary{ + MethodCounts: make(map[string]int), + } + + if resp.DeliverTxs != nil { + for _, txResult := range resp.DeliverTxs { + // Count and decode events + for _, event := range txResult.Events { + eventSummary.EventTypeCounts[event.Type]++ + + // Decode event if sample limit not reached + if decodedEvents, err := decodeConsensusEvent(event); err == nil { + for _, decodedEvent := range decodedEvents { + if len(eventSummary.Events) < 10 { + eventSummary.Events = append(eventSummary.Events, decodedEvent) + } } } } - } - // Decode transaction if present - if len(txResult.Data) > 0 { - if decodedTx, err := decodeConsensusTransaction(txResult.Data); err == nil { - txSummary.MethodCounts[decodedTx.Method]++ - // Only include first few transactions for sample - if len(txSummary.Transactions) < 10 { - txSummary.Transactions = append(txSummary.Transactions, decodedTx) + // Decode transaction if present + if len(txResult.Data) > 0 { + if decodedTx, err := decodeConsensusTransaction(txResult.Data); err == nil { + txSummary.MethodCounts[decodedTx.Method]++ + // Only include first few transactions for sample + if len(txSummary.Transactions) < 10 { + txSummary.Transactions = append(txSummary.Transactions, decodedTx) + } } } } } - } - // Events from BeginBlock - if resp.BeginBlock != nil { - for _, event := range resp.BeginBlock.Events { - eventSummary.EventTypeCounts[event.Type]++ + // Events from BeginBlock + if resp.BeginBlock != nil { + for _, event := range resp.BeginBlock.Events { + eventSummary.EventTypeCounts[event.Type]++ - // Decode event if sample limit not reached - if decodedEvents, err := decodeConsensusEvent(event); err == nil { - for _, decodedEvent := range decodedEvents { - if len(eventSummary.Events) < 10 { - eventSummary.Events = append(eventSummary.Events, decodedEvent) + // Decode event if sample limit not reached + if decodedEvents, err := decodeConsensusEvent(event); err == nil { + for _, decodedEvent := range decodedEvents { + if len(eventSummary.Events) < 10 { + eventSummary.Events = append(eventSummary.Events, decodedEvent) + } } } } } - } - // Events from EndBlock - if resp.EndBlock != nil { - for _, event := range resp.EndBlock.Events { - eventSummary.EventTypeCounts[event.Type]++ + // Events from EndBlock + if resp.EndBlock != nil { + for _, event := range resp.EndBlock.Events { + eventSummary.EventTypeCounts[event.Type]++ - // Decode event if sample limit not reached - if decodedEvents, err := decodeConsensusEvent(event); err == nil { - for _, decodedEvent := range decodedEvents { - if len(eventSummary.Events) < 10 { - eventSummary.Events = append(eventSummary.Events, decodedEvent) + // Decode event if sample limit not reached + if decodedEvents, err := decodeConsensusEvent(event); err == nil { + for _, decodedEvent := range decodedEvents { + if len(eventSummary.Events) < 10 { + eventSummary.Events = append(eventSummary.Events, decodedEvent) + } } } } } - } - if len(txSummary.MethodCounts) > 0 { - abciResponseInfo.TransactionSummary = txSummary - } + if len(txSummary.MethodCounts) > 0 { + abciResponseInfo.TransactionSummary = txSummary + } - if len(eventSummary.EventTypeCounts) > 0 { - abciResponseInfo.EventSummary = eventSummary - } + if len(eventSummary.EventTypeCounts) > 0 { + abciResponseInfo.EventSummary = eventSummary + } - info.ABCIResponse = abciResponseInfo + info.ABCIResponse = abciResponseInfo + } + } else { + info.RawError = fmt.Sprintf("failed to decode as CometBFT or Tendermint v0.34 schema: %v", err) } - } else { - info.RawError = fmt.Sprintf("failed to decode ABCI responses: %v", err) } case "consensus_params": var params tmConsensusParams if err := proto.Unmarshal(value, ¶ms); err == nil { - // Validate that unmarshal decoded meaningful data - if params.Block == nil && params.Evidence == nil && params.Validator == nil && params.Version == nil { - info.RawError = "protobuf unmarshal succeeded but all fields are nil (schema mismatch)" - } else { - info.ConsensusParams = ¶ms - } + // Fields are non-nullable structs, so always initialized + info.ConsensusParams = ¶ms } else { info.RawError = fmt.Sprintf("failed to decode consensus params: %v", err) } diff --git a/badgerdb-sampler/decode_evm.go b/badgerdb-sampler/decode_evm.go index 7c53c14..e4145ff 100644 --- a/badgerdb-sampler/decode_evm.go +++ b/badgerdb-sampler/decode_evm.go @@ -12,6 +12,8 @@ import ( // decodeEVMData decodes EVM module storage data. // See: _oasis-sdk/runtime-sdk/modules/evm/src/state.rs func decodeEVMData(module string, key []byte, value []byte) *EVMDataInfo { + info := &EVMDataInfo{} + // Module format: "evm:subtype" where subtype is extracted from key prefix // The full key after module name starts with the storage type prefix @@ -26,84 +28,83 @@ func decodeEVMData(module string, key []byte, value []byte) *EVMDataInfo { } if moduleNameEnd >= len(key) { - return nil // No data after module name + info.RawError = "no data after module name" + return info } subKey := key[moduleNameEnd:] if len(subKey) < 1 { - return nil + info.RawError = "insufficient subkey data" + return info } - evmInfo := &EVMDataInfo{} storagePrefix := subKey[0] data := subKey[1:] switch storagePrefix { case 0x01: // CODES: evm + 0x01 + H160 (address) - evmInfo.StorageType = "code" + info.StorageType = "code" if len(data) >= 20 { - evmInfo.Address = "0x" + hex.EncodeToString(data[:20]) + info.Address = "0x" + hex.EncodeToString(data[:20]) data = data[20:] } // Value is contract bytecode - evmInfo.CodeSize = len(value) + info.CodeSize = len(value) if len(value) > 0 { - evmInfo.CodeHex = formatRawValue(value, TruncateLongLen) + info.CodeHex = formatRawValue(value, TruncateLongLen) } case 0x02: // STORAGES: evm + 0x02 + H160 (address) + H256 (slot) - evmInfo.StorageType = "storage" + info.StorageType = "storage" if len(data) >= 20 { - evmInfo.Address = "0x" + hex.EncodeToString(data[:20]) + info.Address = "0x" + hex.EncodeToString(data[:20]) data = data[20:] if len(data) >= 32 { - evmInfo.StorageSlot = formatRawValue(data[:32], TruncateHashLen) + info.StorageSlot = formatRawValue(data[:32], TruncateHashLen) } } // Value is H256 storage value if len(value) == 32 { - evmInfo.StorageValue = formatRawValue(value, TruncateLongLen) + info.StorageValue = formatRawValue(value, TruncateLongLen) } case 0x03: // BLOCK_HASHES: evm + 0x03 + RuntimeHeight (uint64 BE) - evmInfo.StorageType = "block_hash" + info.StorageType = "block_hash" if len(data) >= 8 { - evmInfo.RuntimeHeight = binary.BigEndian.Uint64(data[:8]) + info.RuntimeHeight = binary.BigEndian.Uint64(data[:8]) } // Value is H256 block hash if len(value) == 32 { - evmInfo.BlockHash = formatRawValue(value, TruncateHashLen) + info.BlockHash = formatRawValue(value, TruncateHashLen) } case 0x04: // CONFIDENTIAL_STORAGES: evm + 0x04 + H160 (address) + H256 (slot) - evmInfo.StorageType = "confidential_storage" + info.StorageType = "confidential_storage" if len(data) >= 20 { - evmInfo.Address = "0x" + hex.EncodeToString(data[:20]) + info.Address = "0x" + hex.EncodeToString(data[:20]) data = data[20:] if len(data) >= 32 { - evmInfo.StorageSlot = formatRawValue(data[:32], TruncateHashLen) + info.StorageSlot = formatRawValue(data[:32], TruncateHashLen) } } // Value is encrypted - we can only show size if len(value) > 0 { - evmInfo.StorageValue = fmt.Sprintf("", len(value)) + info.StorageValue = fmt.Sprintf("", len(value)) } default: - return nil // Unknown EVM storage type + info.RawError = fmt.Sprintf("unknown EVM storage type: 0x%02x", storagePrefix) + return info } - return evmInfo + return info } // decodeEVMEvent decodes an EVM Log event from CBOR-encoded value. // See: _oasis-sdk/runtime-sdk/modules/evm/src/lib.rs:253-263 // Returns (info, error) where error is a parsing error, not an execution error. func decodeEVMEvent(value []byte) *EVMEventInfo { - info := &EVMEventInfo{ - RawDump: formatRawValue(value, TruncateLongLen), - RawSize: len(value), - } + info := &EVMEventInfo{} // Value is CBOR: [event_code, {address, topics, data}] var eventWrapper []interface{} @@ -158,10 +159,7 @@ func decodeEVMEvent(value []byte) *EVMEventInfo { // decodeEVMTxInput decodes EVM transaction input artifacts. func decodeEVMTxInput(key []byte, value []byte) *EVMTxInputInfo { - info := &EVMTxInputInfo{ - RawDump: formatRawValue(value, TruncateLongLen), - RawSize: len(value), - } + info := &EVMTxInputInfo{} if len(key) >= 33 { info.TxHash = formatRawValue(key[1:33], TruncateHashLen) @@ -250,8 +248,37 @@ func decodeEVMTxInput(key []byte, value []byte) *EVMTxInputInfo { bodyBytes = bodyVal } } else { - info.RawError = "nested map format: method missing" - return info + // Fallback A: Re-marshal map and try decoding as cborRuntimeCallMapFormat + if marshaled, err := cbor.Marshal(nestedMap); err == nil { + var callMap cborRuntimeCallMapFormat + if err := cbor.Unmarshal(marshaled, &callMap); err == nil { + method = callMap.Method + bodyBytes = callMap.Body + } else { + info.RawError = "nested map format: method missing" + return info + } + } else { + info.RawError = "nested map format: method missing" + return info + } + } + } else if nestedBytes, ok := nestedArray[0].([]byte); ok { + // Fallback B: Nested bytes - unwrap and decode + var callMap cborRuntimeCallMapFormat + if err := cbor.Unmarshal(nestedBytes, &callMap); err == nil { + method = callMap.Method + bodyBytes = callMap.Body + } else { + // Try array format as second attempt + var callArray cborRuntimeCallArrayFormat + if err := cbor.Unmarshal(nestedBytes, &callArray); err == nil { + method = callArray.Method + bodyBytes = callArray.Body + } else { + info.RawError = "nested array format: unable to decode nested bytes" + return info + } } } else { info.RawError = "nested array format: invalid element type" @@ -297,10 +324,7 @@ func decodeEVMTxInput(key []byte, value []byte) *EVMTxInputInfo { // decodeEVMTransactionFromCBOR decodes EVM transaction from raw CBOR bytes. // See: _oasis-sdk/runtime-sdk/modules/evm/src/types.rs for transaction structure func decodeEVMTransactionFromCBOR(bodyBytes []byte, isCreate bool) *EVMTransactionInfo { - info := &EVMTransactionInfo{ - RawDump: formatRawValue(bodyBytes, TruncateLongLen), - RawSize: len(bodyBytes), - } + info := &EVMTransactionInfo{} if isCreate { info.Type = "create" @@ -367,10 +391,7 @@ func decodeEVMTransactionFromCBOR(bodyBytes []byte, isCreate bool) *EVMTransacti // decodeEVMTxOutput decodes EVM transaction output artifacts. // Execution errors (transaction failures) are stored in info.ErrorExecution field. func decodeEVMTxOutput(value []byte) *EVMTxOutputInfo { - info := &EVMTxOutputInfo{ - RawDump: formatRawValue(value, TruncateLongLen), - RawSize: len(value), - } + info := &EVMTxOutputInfo{} var oa cborRuntimeOutputArtifacts if err := cbor.Unmarshal(value, &oa); err != nil { diff --git a/badgerdb-sampler/decode_runtime.go b/badgerdb-sampler/decode_runtime.go index e3ca621..3bb2e7f 100644 --- a/badgerdb-sampler/decode_runtime.go +++ b/badgerdb-sampler/decode_runtime.go @@ -80,6 +80,14 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) *RuntimeMkvsValueInfo info.NodeType = "leaf" data := value[1:] + // Try pre-v21.1.0 format (skip 8-byte Version field) + if len(data) >= 10 { + testKeySize := int(binary.LittleEndian.Uint16(data[8:10])) + if testKeySize > 0 && testKeySize <= 10000 && len(data) >= 10+testKeySize+4 { + data = data[8:] + } + } + if len(data) < 2 { info.RawError = "key length missing" return info @@ -129,6 +137,14 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) *RuntimeMkvsValueInfo info.NodeType = "internal" data := value[1:] + // Try pre-v21.1.0 format (skip 8-byte Version field) + if len(data) >= 10 { + testLabelBits := binary.LittleEndian.Uint16(data[8:10]) + if testLabelBits <= 2048 && len(data) >= 10+(int(testLabelBits)+7)/8+1 { + data = data[8:] + } + } + if len(data) < 2 { info.RawError = "label bits missing" return info @@ -310,6 +326,31 @@ func decodeRuntimeLeafValue(module string, key []byte, value []byte) *RuntimeLea ValueType: "unknown", // default } + // Extract module name and subprefix for format lookup + modName := module + if idx := strings.Index(module, ":"); idx > 0 { + modName = module[:idx] + } + var subPrefix byte + if len(modName) < len(key) { + subPrefix = key[len(modName)] + } + + // Check lookup table for deterministic format handling + format, exists := GetRuntimeModuleStateFormat(modName, subPrefix) + if exists { + info.ValueType = format.Description + switch format.Format { + case "binary": + info.CBOR = fmt.Sprintf("%s (%d bytes)", format.Type, len(value)) + return info + case "wasm": + info.CBOR = fmt.Sprintf("wasm bytecode (%d bytes)", len(value)) + return info + // case "cbor": fall through to existing CBOR decode logic below + } + } + // Check for EVM module data if len(module) >= 3 && module[:3] == "evm" { evmInfo := decodeEVMData(module, key, value) diff --git a/badgerdb-sampler/main.go b/badgerdb-sampler/main.go index 50a57e2..e7cd52d 100644 --- a/badgerdb-sampler/main.go +++ b/badgerdb-sampler/main.go @@ -215,53 +215,75 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { keyInfo := decodeRuntimeHistoryKey(key) sample.Key = keyInfo sample.KeyType = keyInfo.KeyType + default: + log.Fatalf("Unsupported database type: %s", stats.DatabaseType) } // Fetch and decode value switch stats.DatabaseType { case "consensus-blockstore": + valSize := int(item.ValueSize()) val, err := item.ValueCopy(nil) if err != nil { - sample.Value = &ConsensusBlockstoreValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err)} + sample.Value = &ConsensusBlockstoreValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err), RawSize: valSize} + } else if len(val) != valSize { + sample.Value = &ConsensusBlockstoreValueInfo{RawError: fmt.Sprintf("value size mismatch (got: %d)", len(val)), RawSize: valSize} } else { sample.Value = decodeConsensusBlockstoreValue(sample.KeyType, val) sample.Timestamp = sample.Value.(*ConsensusBlockstoreValueInfo).Timestamp } case "consensus-evidence": + valSize := int(item.ValueSize()) val, err := item.ValueCopy(nil) if err != nil { - sample.Value = &ConsensusEvidenceValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err)} + sample.Value = &ConsensusEvidenceValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err), RawSize: valSize} + } else if len(val) != valSize { + sample.Value = &ConsensusEvidenceValueInfo{RawError: fmt.Sprintf("value size mismatch (got: %d)", len(val)), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} } else { sample.Value = decodeConsensusEvidenceValue(sample.KeyType, val) } case "consensus-mkvs": + valSize := int(item.ValueSize()) val, err := item.ValueCopy(nil) if err != nil { - sample.Value = &ConsensusMkvsValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err)} + sample.Value = &ConsensusMkvsValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err), RawSize: valSize} + } else if len(val) != valSize { + sample.Value = &ConsensusMkvsValueInfo{RawError: fmt.Sprintf("value size mismatch (got: %d)", len(val)), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} } else { sample.Value = decodeConsensusMkvsValue(sample.KeyType, val) } case "consensus-state": + valSize := int(item.ValueSize()) val, err := item.ValueCopy(nil) if err != nil { - sample.Value = &ConsensusStateValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err)} + sample.Value = &ConsensusStateValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err), RawSize: valSize} + } else if len(val) != valSize { + sample.Value = &ConsensusStateValueInfo{RawError: fmt.Sprintf("value size mismatch (got: %d)", len(val)), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} } else { sample.Value = decodeConsensusStateValue(sample.KeyType, val) } case "runtime-mkvs": + valSize := int(item.ValueSize()) val, err := item.ValueCopy(nil) if err != nil { - sample.Value = &RuntimeMkvsValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err)} + sample.Value = &RuntimeMkvsValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err), RawSize: valSize} + } else if len(val) != valSize { + sample.Value = &RuntimeMkvsValueInfo{RawError: fmt.Sprintf("value size mismatch (got: %d)", len(val)), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} } else { sample.Value = decodeRuntimeMkvsValue(sample.KeyType, val) } case "runtime-history": + valSize := int(item.ValueSize()) val, err := item.ValueCopy(nil) if err != nil { - sample.Value = &RuntimeHistoryValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err)} + sample.Value = &RuntimeHistoryValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err), RawSize: valSize} + } else if len(val) != valSize { + sample.Value = &RuntimeHistoryValueInfo{RawError: fmt.Sprintf("value size mismatch (got: %d)", len(val)), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} } else { sample.Value = decodeRuntimeHistoryValue(sample.KeyType, val) } + default: + log.Fatalf("Unsupported database type: %s", stats.DatabaseType) } stats.Samples = append(stats.Samples, sample) diff --git a/badgerdb-sampler/types_consensus.go b/badgerdb-sampler/types_consensus.go index 8df6b1a..cc16ea0 100644 --- a/badgerdb-sampler/types_consensus.go +++ b/badgerdb-sampler/types_consensus.go @@ -86,7 +86,8 @@ type ConsensusCommitInfo struct { // ============================================================================= // ConsensusEvidenceKeyInfo represents a decoded consensus-evidence key. -// See: tendermint/store/evidence/pool.go for key formats +// Key format: dbVersion (0x01) + prefix (0x00=committed, 0x01=pending) + "HEIGHT_HEX/HASH_HEX" +// See: cometbft/evidence/pool.go for key formats (keyCommitted, keyPending) type ConsensusEvidenceKeyInfo struct { // Raw fields KeyDump string `json:"key_dump,omitempty"` @@ -94,25 +95,31 @@ type ConsensusEvidenceKeyInfo struct { KeyError string `json:"key_error,omitempty"` // Decoded fields - KeyType string `json:"key_type"` - PrefixByte byte `json:"prefix_byte,omitempty"` + KeyType string `json:"key_type"` // "committed", "pending", "type_XX" + PrefixByte byte `json:"prefix_byte,omitempty"` + ConsensusHeight int64 `json:"consensus_height,omitempty"` // Extracted from key suffix + Hash string `json:"hash,omitempty"` // Evidence hash from key suffix } // ConsensusEvidenceValueInfo represents a decoded consensus-evidence value. -// See: tendermint/proto/tendermint/types/evidence.proto (DuplicateVoteEvidence) +// Committed evidence stores Int64Value (height only), pending stores full Evidence protobuf. +// See: cometbft/evidence/pool.go (addPendingEvidence stores full evidence, markEvidenceAsCommitted stores height) +// See: tendermint/proto/tendermint/types/evidence.proto (DuplicateVoteEvidence, LightClientAttackEvidence) type ConsensusEvidenceValueInfo struct { // Raw fields - RawDump string `json:"raw_dump,omitempty"` - RawSize int `json:"raw_size"` - RawError string `json:"raw_error,omitempty"` + RawDump string `json:"raw_dump,omitempty"` + RawSize int `json:"raw_size"` + RawError string `json:"raw_error,omitempty"` + SchemaVersion string `json:"schema_version,omitempty"` // "cb-v0.37", "tm-v0.34", "unknown" // Decoded fields - EvidenceType string `json:"evidence_type,omitempty"` // "duplicate_vote", "light_client_attack", "unknown" - VoteAHeight int64 `json:"vote_a_height,omitempty"` - VoteBHeight int64 `json:"vote_b_height,omitempty"` - TotalVotingPower int64 `json:"total_voting_power,omitempty"` - ValidatorPower int64 `json:"validator_power,omitempty"` - Timestamp string `json:"timestamp,omitempty"` // RFC3339 format + EvidenceType string `json:"evidence_type,omitempty"` // "committed_marker", "duplicate_vote", "light_client_attack", "unknown" + CommittedHeight int64 `json:"committed_height,omitempty"` // For committed evidence (Int64Value) + VoteAHeight int64 `json:"vote_a_height,omitempty"` // For pending DuplicateVoteEvidence + VoteBHeight int64 `json:"vote_b_height,omitempty"` // For pending DuplicateVoteEvidence + TotalVotingPower int64 `json:"total_voting_power,omitempty"` // For pending evidence + ValidatorPower int64 `json:"validator_power,omitempty"` // For pending evidence + Timestamp string `json:"timestamp,omitempty"` // RFC3339 format (for pending evidence) } // ============================================================================= @@ -169,6 +176,7 @@ type ConsensusMkvsLeafInfo struct { // Decoded value ValueType string `json:"value_type,omitempty"` Value interface{} `json:"value,omitempty"` + CBOR string `json:"cbor,omitempty"` // CBOR type description or format hint } // ConsensusMkvsInternalInfo represents a decoded MKVS InternalNode. @@ -202,9 +210,10 @@ type ConsensusStateKeyInfo struct { // See: tendermint/proto/tendermint/state/types.proto (various state types) type ConsensusStateValueInfo struct { // Raw fields - RawDump string `json:"raw_dump,omitempty"` - RawSize int `json:"raw_size"` - RawError string `json:"raw_error,omitempty"` + RawDump string `json:"raw_dump,omitempty"` + RawSize int `json:"raw_size"` + RawError string `json:"raw_error,omitempty"` + SchemaVersion string `json:"schema_version,omitempty"` // "cometbft", "tendermint-v0.34", "unknown" // Decoded content - only ONE populated ABCIResponse *ConsensusABCIResponseInfo `json:"abci_response,omitempty"` // For abci_responses @@ -439,34 +448,30 @@ type cborConsensusReclaimEscrowEvent struct { // ============================================================================= // Source: github.com/tendermint/tendermint v0.34.21 // These types are used only for protobuf deserialization - no tendermint business logic is needed. - -// protoMessage is a common base type that implements proto.Message interface. -// All tendermint types embed this to satisfy proto.Unmarshal requirements. -type protoMessage struct{} - -func (protoMessage) Reset() {} -func (protoMessage) String() string { return "" } -func (protoMessage) ProtoMessage() {} +// For proto.Unmarshal() to work types need to implement proto.Message interface. // tmBlockStoreState represents BlockStoreState from tendermint/proto/tendermint/store/types.proto type tmBlockStoreState struct { - protoMessage Base int64 `protobuf:"varint,1,opt,name=base,proto3"` Height int64 `protobuf:"varint,2,opt,name=height,proto3"` } +func (m *tmBlockStoreState) Reset() { *m = tmBlockStoreState{} } +func (m *tmBlockStoreState) String() string { return "" } +func (*tmBlockStoreState) ProtoMessage() {} // tmBlockMeta represents BlockMeta from tendermint/proto/tendermint/types/types.proto type tmBlockMeta struct { - protoMessage BlockID tmBlockID `protobuf:"bytes,1,opt,name=block_id,json=blockId,proto3"` BlockSize int64 `protobuf:"varint,2,opt,name=block_size,json=blockSize,proto3"` Header tmHeader `protobuf:"bytes,3,opt,name=header,proto3"` NumTxs int64 `protobuf:"varint,4,opt,name=num_txs,json=numTxs,proto3"` } +func (m *tmBlockMeta) Reset() { *m = tmBlockMeta{} } +func (m *tmBlockMeta) String() string { return "" } +func (*tmBlockMeta) ProtoMessage() {} // tmBlockID from tendermint/proto/tendermint/types/types.proto type tmBlockID struct { - protoMessage Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3"` PartSetHeader tmPartSetHeader `protobuf:"bytes,2,opt,name=part_set_header,json=partSetHeader,proto3"` } @@ -474,7 +479,6 @@ type tmBlockID struct { // tmPartSetHeader from tendermint/proto/tendermint/types/types.proto type tmPartSetHeader struct { - protoMessage Total uint32 `protobuf:"varint,1,opt,name=total,proto3"` Hash []byte `protobuf:"bytes,2,opt,name=hash,proto3"` } @@ -482,7 +486,6 @@ type tmPartSetHeader struct { // tmHeader from tendermint/proto/tendermint/types/types.proto type tmHeader struct { - protoMessage Version tmConsensus `protobuf:"bytes,1,opt,name=version,proto3"` ChainID string `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3"` Height int64 `protobuf:"varint,3,opt,name=height,proto3"` @@ -502,7 +505,6 @@ type tmHeader struct { // tmConsensus from tendermint/proto/tendermint/types/types.proto type tmConsensus struct { - protoMessage Block uint64 `protobuf:"varint,1,opt,name=block,proto3"` App uint64 `protobuf:"varint,2,opt,name=app,proto3"` } @@ -510,16 +512,17 @@ type tmConsensus struct { // tmPart from tendermint/proto/tendermint/types/types.proto type tmPart struct { - protoMessage Index uint32 `protobuf:"varint,1,opt,name=index,proto3"` Bytes []byte `protobuf:"bytes,2,opt,name=bytes,proto3"` Proof tmProof `protobuf:"bytes,3,opt,name=proof,proto3"` } +func (m *tmPart) Reset() { *m = tmPart{} } +func (m *tmPart) String() string { return "" } +func (*tmPart) ProtoMessage() {} // tmProof from tendermint/proto/tendermint/types/types.proto type tmProof struct { - protoMessage Total int64 `protobuf:"varint,1,opt,name=total,proto3"` Index int64 `protobuf:"varint,2,opt,name=index,proto3"` LeafHash []byte `protobuf:"bytes,3,opt,name=leaf_hash,json=leafHash,proto3"` @@ -529,17 +532,18 @@ type tmProof struct { // tmCommit from tendermint/proto/tendermint/types/types.proto type tmCommit struct { - protoMessage Height int64 `protobuf:"varint,1,opt,name=height,proto3"` Round int32 `protobuf:"varint,2,opt,name=round,proto3"` BlockID tmBlockID `protobuf:"bytes,3,opt,name=block_id,json=blockId,proto3"` Signatures []tmCommitSig `protobuf:"bytes,4,rep,name=signatures,proto3"` } +func (m *tmCommit) Reset() { *m = tmCommit{} } +func (m *tmCommit) String() string { return "" } +func (*tmCommit) ProtoMessage() {} // tmCommitSig from tendermint/proto/tendermint/types/types.proto type tmCommitSig struct { - protoMessage BlockIDFlag int32 `protobuf:"varint,1,opt,name=block_id_flag,json=blockIdFlag,proto3"` ValidatorAddress []byte `protobuf:"bytes,2,opt,name=validator_address,json=validatorAddress,proto3"` Timestamp time.Time `protobuf:"bytes,3,opt,name=timestamp,proto3,stdtime"` @@ -547,20 +551,31 @@ type tmCommitSig struct { } +// tmInt64Value from gogoproto/protobuf/wrappers.proto +// Used by Tendermint to store committed evidence (just the height) +type tmInt64Value struct { + Value int64 `protobuf:"varint,1,opt,name=value,proto3"` +} +func (m *tmInt64Value) Reset() { *m = tmInt64Value{} } +func (m *tmInt64Value) String() string { return "" } +func (*tmInt64Value) ProtoMessage() {} + + // tmDuplicateVoteEvidence from tendermint/proto/tendermint/types/evidence.proto type tmDuplicateVoteEvidence struct { - protoMessage VoteA *tmVote `protobuf:"bytes,1,opt,name=vote_a,json=voteA,proto3"` VoteB *tmVote `protobuf:"bytes,2,opt,name=vote_b,json=voteB,proto3"` TotalVotingPower int64 `protobuf:"varint,3,opt,name=total_voting_power,json=totalVotingPower,proto3"` ValidatorPower int64 `protobuf:"varint,4,opt,name=validator_power,json=validatorPower,proto3"` Timestamp time.Time `protobuf:"bytes,5,opt,name=timestamp,proto3,stdtime"` } +func (m *tmDuplicateVoteEvidence) Reset() { *m = tmDuplicateVoteEvidence{} } +func (m *tmDuplicateVoteEvidence) String() string { return "" } +func (*tmDuplicateVoteEvidence) ProtoMessage() {} // tmVote from tendermint/proto/tendermint/types/types.proto type tmVote struct { - protoMessage Type int32 `protobuf:"varint,1,opt,name=type,proto3"` Height int64 `protobuf:"varint,2,opt,name=height,proto3"` Round int32 `protobuf:"varint,3,opt,name=round,proto3"` @@ -574,18 +589,19 @@ type tmVote struct { // tmLightClientAttackEvidence from tendermint/proto/tendermint/types/evidence.proto type tmLightClientAttackEvidence struct { - protoMessage ConflictingBlock *tmLightBlock `protobuf:"bytes,1,opt,name=conflicting_block,json=conflictingBlock,proto3"` CommonHeight int64 `protobuf:"varint,2,opt,name=common_height,json=commonHeight,proto3"` ByzantineValidators []*tmValidator `protobuf:"bytes,3,rep,name=byzantine_validators,json=byzantineValidators,proto3"` TotalVotingPower int64 `protobuf:"varint,4,opt,name=total_voting_power,json=totalVotingPower,proto3"` Timestamp time.Time `protobuf:"bytes,5,opt,name=timestamp,proto3,stdtime"` } +func (m *tmLightClientAttackEvidence) Reset() { *m = tmLightClientAttackEvidence{} } +func (m *tmLightClientAttackEvidence) String() string { return "" } +func (*tmLightClientAttackEvidence) ProtoMessage() {} // tmLightBlock from tendermint/proto/tendermint/types/types.proto type tmLightBlock struct { - protoMessage SignedHeader *tmSignedHeader `protobuf:"bytes,1,opt,name=signed_header,json=signedHeader,proto3"` ValidatorSet *tmValidatorSet `protobuf:"bytes,2,opt,name=validator_set,json=validatorSet,proto3"` } @@ -593,7 +609,6 @@ type tmLightBlock struct { // tmSignedHeader from tendermint/proto/tendermint/types/types.proto type tmSignedHeader struct { - protoMessage Header *tmHeader `protobuf:"bytes,1,opt,name=header,proto3"` Commit *tmCommit `protobuf:"bytes,2,opt,name=commit,proto3"` } @@ -601,16 +616,17 @@ type tmSignedHeader struct { // tmValidatorSet from tendermint/proto/tendermint/types/validator.proto type tmValidatorSet struct { - protoMessage Validators []*tmValidator `protobuf:"bytes,1,rep,name=validators,proto3"` Proposer *tmValidator `protobuf:"bytes,2,opt,name=proposer,proto3"` TotalVotingPower int64 `protobuf:"varint,3,opt,name=total_voting_power,json=totalVotingPower,proto3"` } +func (m *tmValidatorSet) Reset() { *m = tmValidatorSet{} } +func (m *tmValidatorSet) String() string { return "" } +func (*tmValidatorSet) ProtoMessage() {} // tmValidator from tendermint/proto/tendermint/types/validator.proto type tmValidator struct { - protoMessage Address []byte `protobuf:"bytes,1,opt,name=address,proto3"` PubKey []byte `protobuf:"bytes,2,opt,name=pub_key,json=pubKey,proto3"` VotingPower int64 `protobuf:"varint,3,opt,name=voting_power,json=votingPower,proto3"` @@ -620,16 +636,16 @@ type tmValidator struct { // tmABCIResponses from tendermint/proto/tendermint/state/types.proto type tmABCIResponses struct { - protoMessage DeliverTxs []*tmResponseDeliverTx `protobuf:"bytes,1,rep,name=deliver_txs,json=deliverTxs,proto3"` EndBlock *tmResponseEndBlock `protobuf:"bytes,2,opt,name=end_block,json=endBlock,proto3"` BeginBlock *tmResponseBeginBlock `protobuf:"bytes,3,opt,name=begin_block,json=beginBlock,proto3"` } - +func (m *tmABCIResponses) Reset() { *m = tmABCIResponses{} } +func (m *tmABCIResponses) String() string { return "" } +func (*tmABCIResponses) ProtoMessage() {} // tmResponseDeliverTx from tendermint/proto/tendermint/abci/types.proto type tmResponseDeliverTx struct { - protoMessage Code uint32 `protobuf:"varint,1,opt,name=code,proto3"` Data []byte `protobuf:"bytes,2,opt,name=data,proto3"` Log string `protobuf:"bytes,3,opt,name=log,proto3"` @@ -640,17 +656,14 @@ type tmResponseDeliverTx struct { Codespace string `protobuf:"bytes,8,opt,name=codespace,proto3"` } - // tmResponseBeginBlock from tendermint/proto/tendermint/abci/types.proto type tmResponseBeginBlock struct { - protoMessage Events []tmEvent `protobuf:"bytes,1,rep,name=events,proto3"` } // tmResponseEndBlock from tendermint/proto/tendermint/abci/types.proto type tmResponseEndBlock struct { - protoMessage ValidatorUpdates []tmValidatorUpdate `protobuf:"bytes,1,rep,name=validator_updates,json=validatorUpdates,proto3"` ConsensusParamUpdates *tmConsensusParams `protobuf:"bytes,2,opt,name=consensus_param_updates,json=consensusParamUpdates,proto3"` Events []tmEvent `protobuf:"bytes,3,rep,name=events,proto3"` @@ -659,7 +672,6 @@ type tmResponseEndBlock struct { // tmEvent from tendermint/proto/tendermint/abci/types.proto type tmEvent struct { - protoMessage Type string `protobuf:"bytes,1,opt,name=type,proto3"` Attributes []tmEventAttribute `protobuf:"bytes,2,rep,name=attributes,proto3"` } @@ -667,7 +679,6 @@ type tmEvent struct { // tmEventAttribute from tendermint/proto/tendermint/abci/types.proto type tmEventAttribute struct { - protoMessage Key []byte `protobuf:"bytes,1,opt,name=key,proto3"` Value []byte `protobuf:"bytes,2,opt,name=value,proto3"` Index bool `protobuf:"varint,3,opt,name=index,proto3"` @@ -676,7 +687,6 @@ type tmEventAttribute struct { // tmValidatorUpdate from tendermint/proto/tendermint/abci/types.proto type tmValidatorUpdate struct { - protoMessage PubKey []byte `protobuf:"bytes,1,opt,name=pub_key,json=pubKey,proto3"` Power int64 `protobuf:"varint,2,opt,name=power,proto3"` } @@ -684,17 +694,18 @@ type tmValidatorUpdate struct { // tmConsensusParams from tendermint/proto/tendermint/types/params.proto type tmConsensusParams struct { - protoMessage - Block *tmBlockParams `protobuf:"bytes,1,opt,name=block,proto3"` - Evidence *tmEvidenceParams `protobuf:"bytes,2,opt,name=evidence,proto3"` - Validator *tmValidatorParams `protobuf:"bytes,3,opt,name=validator,proto3"` - Version *tmVersionParams `protobuf:"bytes,4,opt,name=version,proto3"` + Block tmBlockParams `protobuf:"bytes,1,opt,name=block,proto3"` + Evidence tmEvidenceParams `protobuf:"bytes,2,opt,name=evidence,proto3"` + Validator tmValidatorParams `protobuf:"bytes,3,opt,name=validator,proto3"` + Version tmVersionParams `protobuf:"bytes,4,opt,name=version,proto3"` } +func (m *tmConsensusParams) Reset() { *m = tmConsensusParams{} } +func (m *tmConsensusParams) String() string { return "" } +func (*tmConsensusParams) ProtoMessage() {} // tmBlockParams from tendermint/proto/tendermint/types/params.proto type tmBlockParams struct { - protoMessage MaxBytes int64 `protobuf:"varint,1,opt,name=max_bytes,json=maxBytes,proto3"` MaxGas int64 `protobuf:"varint,2,opt,name=max_gas,json=maxGas,proto3"` } @@ -702,7 +713,6 @@ type tmBlockParams struct { // tmEvidenceParams from tendermint/proto/tendermint/types/params.proto type tmEvidenceParams struct { - protoMessage MaxAgeNumBlocks int64 `protobuf:"varint,1,opt,name=max_age_num_blocks,json=maxAgeNumBlocks,proto3"` MaxAgeDuration int64 `protobuf:"varint,2,opt,name=max_age_duration,json=maxAgeDuration,proto3"` MaxBytes int64 `protobuf:"varint,3,opt,name=max_bytes,json=maxBytes,proto3"` @@ -711,24 +721,191 @@ type tmEvidenceParams struct { // tmValidatorParams from tendermint/proto/tendermint/types/params.proto type tmValidatorParams struct { - protoMessage PubKeyTypes []string `protobuf:"bytes,1,rep,name=pub_key_types,json=pubKeyTypes,proto3"` } // tmVersionParams from tendermint/proto/tendermint/types/params.proto type tmVersionParams struct { - protoMessage AppVersion uint64 `protobuf:"varint,1,opt,name=app_version,json=appVersion,proto3"` } // tmState from tendermint/proto/tendermint/state/types.proto type tmState struct { - protoMessage Version uint64 `protobuf:"varint,1,opt,name=version,proto3"` ChainID string `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3"` InitialHeight int64 `protobuf:"varint,3,opt,name=initial_height,json=initialHeight,proto3"` LastBlockHeight int64 `protobuf:"varint,4,opt,name=last_block_height,json=lastBlockHeight,proto3"` } +func (m *tmState) Reset() { *m = tmState{} } +func (m *tmState) String() string { return "" } +func (*tmState) ProtoMessage() {} + +// ============================================================================= +// CometBFT Protobuf Deserialization Types +// ============================================================================= +// Source: github.com/cometbft/cometbft v0.37.x / v0.38.x +// https://github.com/cometbft/cometbft/blob/main/proto/cometbft/abci/v1/types.proto +// https://github.com/cometbft/cometbft/blob/main/proto/cometbft/types/v1/evidence.proto +// For proto.Unmarshal() to work types need to implement proto.Message interface. + +// Type aliases for cb* types identical to tm* types +// Source: cometbft/proto/cometbft/abci/v1/types.proto +type cbEvent = tmEvent +type cbEventAttribute = tmEventAttribute +type cbValidatorUpdate = tmValidatorUpdate +type cbConsensusParams = tmConsensusParams +type cbBlockParams = tmBlockParams +type cbEvidenceParams = tmEvidenceParams +type cbValidatorParams = tmValidatorParams +type cbVersionParams = tmVersionParams +type cbBlockID = tmBlockID +type cbPartSetHeader = tmPartSetHeader +type cbInt64Value = tmInt64Value + +// cbResponseFinalizeBlock from CometBFT ABCI++ (v0.37+) +// Source: cometbft/proto/cometbft/abci/v1/types.proto +type cbResponseFinalizeBlock struct { + Events []cbEvent `protobuf:"bytes,1,rep,name=events,proto3"` + TxResults []*cbExecTxResult `protobuf:"bytes,2,rep,name=tx_results,json=txResults,proto3"` + ValidatorUpdates []cbValidatorUpdate `protobuf:"bytes,3,rep,name=validator_updates,json=validatorUpdates,proto3"` + ConsensusParamUpdates *cbConsensusParams `protobuf:"bytes,4,opt,name=consensus_param_updates,json=consensusParamUpdates,proto3"` + AppHash []byte `protobuf:"bytes,5,opt,name=app_hash,json=appHash,proto3"` +} +func (m *cbResponseFinalizeBlock) Reset() { *m = cbResponseFinalizeBlock{} } +func (m *cbResponseFinalizeBlock) String() string { return "" } +func (*cbResponseFinalizeBlock) ProtoMessage() {} + +// cbExecTxResult from CometBFT ABCI++ (v0.37+) +// Source: cometbft/proto/cometbft/abci/v1/types.proto +type cbExecTxResult struct { + Code uint32 `protobuf:"varint,1,opt,name=code,proto3"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3"` + Log string `protobuf:"bytes,3,opt,name=log,proto3"` + Info string `protobuf:"bytes,4,opt,name=info,proto3"` + GasWanted int64 `protobuf:"varint,5,opt,name=gas_wanted,json=gasWanted,proto3"` + GasUsed int64 `protobuf:"varint,6,opt,name=gas_used,json=gasUsed,proto3"` + Events []cbEvent `protobuf:"bytes,7,rep,name=events,proto3"` + Codespace string `protobuf:"bytes,8,opt,name=codespace,proto3"` +} + +// cbDuplicateVoteEvidence from CometBFT +// Source: cometbft/proto/cometbft/types/v1/evidence.proto +type cbDuplicateVoteEvidence struct { + VoteA *cbVote `protobuf:"bytes,1,opt,name=vote_a,json=voteA,proto3"` + VoteB *cbVote `protobuf:"bytes,2,opt,name=vote_b,json=voteB,proto3"` + TotalVotingPower int64 `protobuf:"varint,3,opt,name=total_voting_power,json=totalVotingPower,proto3"` + ValidatorPower int64 `protobuf:"varint,4,opt,name=validator_power,json=validatorPower,proto3"` + Timestamp time.Time `protobuf:"bytes,5,opt,name=timestamp,proto3,stdtime"` +} +func (m *cbDuplicateVoteEvidence) Reset() { *m = cbDuplicateVoteEvidence{} } +func (m *cbDuplicateVoteEvidence) String() string { return "" } +func (*cbDuplicateVoteEvidence) ProtoMessage() {} + +// cbVote from CometBFT +// Source: cometbft/proto/cometbft/types/v1/types.proto +type cbVote struct { + Type int32 `protobuf:"varint,1,opt,name=type,proto3"` + Height int64 `protobuf:"varint,2,opt,name=height,proto3"` + Round int32 `protobuf:"varint,3,opt,name=round,proto3"` + BlockID *cbBlockID `protobuf:"bytes,4,opt,name=block_id,json=blockId,proto3"` + Timestamp time.Time `protobuf:"bytes,5,opt,name=timestamp,proto3,stdtime"` + ValidatorAddress []byte `protobuf:"bytes,6,opt,name=validator_address,json=validatorAddress,proto3"` + ValidatorIndex int32 `protobuf:"varint,7,opt,name=validator_index,json=validatorIndex,proto3"` + Signature []byte `protobuf:"bytes,8,opt,name=signature,proto3"` +} + + +// ============================================================================= +// Consensus MKVS Format Mappings +// ============================================================================= + +// ConsensusMKVSFormat describes the value format for a consensus MKVS key prefix. +type ConsensusMKVSFormat struct { + Format string // "cbor", "binary", "empty" + Type string // Go type name or description + Description string // Human-readable description +} + +// consensusMKVSFormats maps consensus MKVS key prefixes to their value formats. +// Extracted from _oasis-core/go/consensus/tendermint/apps/*/state/state.go +// This provides deterministic decoding without heuristics. +var consensusMKVSFormats = map[byte]ConsensusMKVSFormat{ + // Registry Module (0x10-0x19) + // See: _oasis-core/go/consensus/tendermint/apps/registry/state/state.go + 0x10: {Format: "cbor", Type: "entity.SignedEntity", Description: "signed entity"}, + 0x11: {Format: "cbor", Type: "node.MultiSignedNode", Description: "signed node"}, + 0x12: {Format: "empty", Type: "index", Description: "node by entity index"}, + 0x13: {Format: "cbor", Type: "registry.Runtime", Description: "runtime"}, + 0x14: {Format: "binary", Type: "signature.PublicKey", Description: "node by consensus address"}, + 0x15: {Format: "cbor", Type: "registry.NodeStatus", Description: "node status"}, + 0x16: {Format: "cbor", Type: "registry.ConsensusParameters", Description: "registry parameters"}, + 0x17: {Format: "binary", Type: "signature.PublicKey", Description: "key map"}, + 0x18: {Format: "cbor", Type: "registry.Runtime", Description: "suspended runtime"}, + 0x19: {Format: "empty", Type: "index", Description: "runtime by entity index"}, + + // Roothash Module (0x20-0x29) + // See: _oasis-core/go/consensus/tendermint/apps/roothash/state/state.go + 0x20: {Format: "cbor", Type: "roothash.RuntimeState", Description: "runtime state"}, + 0x21: {Format: "cbor", Type: "roothash.ConsensusParameters", Description: "roothash parameters"}, + 0x22: {Format: "binary", Type: "common.Namespace", Description: "round timeout queue"}, + 0x24: {Format: "empty", Type: "index", Description: "evidence"}, + 0x25: {Format: "binary", Type: "hash.Hash", Description: "state root"}, + 0x26: {Format: "binary", Type: "hash.Hash", Description: "io root"}, + 0x27: {Format: "cbor", Type: "roothash.RoundResults", Description: "last round results"}, + 0x28: {Format: "cbor", Type: "message.IncomingMessageQueueMeta", Description: "incoming message queue meta"}, + 0x29: {Format: "cbor", Type: "message.IncomingMessage", Description: "incoming message queue"}, + + // Beacon Module (0x40-0x45) + // See: _oasis-core/go/consensus/tendermint/apps/beacon/state/state.go + 0x40: {Format: "cbor", Type: "beacon.EpochTimeState", Description: "epoch current"}, + 0x41: {Format: "cbor", Type: "beacon.EpochTimeState", Description: "epoch future"}, + 0x42: {Format: "binary", Type: "beacon-32bytes", Description: "beacon value"}, + 0x43: {Format: "cbor", Type: "beacon.ConsensusParameters", Description: "beacon parameters"}, + 0x45: {Format: "cbor", Type: "beacon.EpochTime", Description: "pending mock epoch"}, + + // Staking Module (0x50-0x59) + // See: _oasis-core/go/consensus/tendermint/apps/staking/state/state.go + 0x50: {Format: "cbor", Type: "staking.Account", Description: "account"}, + 0x51: {Format: "cbor", Type: "quantity.Quantity", Description: "total supply"}, + 0x52: {Format: "cbor", Type: "quantity.Quantity", Description: "common pool"}, + 0x53: {Format: "cbor", Type: "staking.Delegation", Description: "delegation"}, + 0x54: {Format: "cbor", Type: "staking.DebondingDelegation", Description: "debonding delegation"}, + 0x55: {Format: "empty", Type: "index", Description: "debonding queue"}, + 0x56: {Format: "cbor", Type: "staking.ConsensusParameters", Description: "staking parameters"}, + 0x57: {Format: "cbor", Type: "quantity.Quantity", Description: "last block fees"}, + 0x58: {Format: "cbor", Type: "EpochSigning", Description: "epoch signing"}, + 0x59: {Format: "cbor", Type: "quantity.Quantity", Description: "governance deposits"}, + + // Scheduler Module (0x60-0x63) + // See: _oasis-core/go/consensus/tendermint/apps/scheduler/state/state.go + 0x60: {Format: "cbor", Type: "api.Committee", Description: "committee"}, + 0x61: {Format: "cbor", Type: "map[signature.PublicKey]int64", Description: "current validators"}, + 0x62: {Format: "cbor", Type: "map[signature.PublicKey]int64", Description: "pending validators"}, + 0x63: {Format: "cbor", Type: "api.ConsensusParameters", Description: "scheduler parameters"}, + + // Keymanager Module (0x70) + // See: _oasis-core/go/consensus/tendermint/apps/keymanager/state/state.go + 0x70: {Format: "cbor", Type: "api.Status", Description: "keymanager status"}, + + // Governance Module (0x80-0x85) + // See: _oasis-core/go/consensus/tendermint/apps/governance/state/state.go + 0x80: {Format: "cbor", Type: "uint64", Description: "next proposal identifier"}, + 0x81: {Format: "cbor", Type: "governance.Proposal", Description: "proposals"}, + 0x82: {Format: "empty", Type: "index", Description: "active proposals"}, + 0x83: {Format: "cbor", Type: "governance.Vote", Description: "votes"}, + 0x84: {Format: "empty", Type: "index", Description: "pending upgrades"}, + 0x85: {Format: "cbor", Type: "governance.ConsensusParameters", Description: "governance parameters"}, +} + +// GetConsensusMKVSFormat returns the format metadata for a consensus MKVS key. +// Returns (format, true) if the key prefix is known, (empty, false) otherwise. +func GetConsensusMKVSFormat(key []byte) (ConsensusMKVSFormat, bool) { + if len(key) == 0 { + return ConsensusMKVSFormat{}, false + } + format, exists := consensusMKVSFormats[key[0]] + return format, exists +} diff --git a/badgerdb-sampler/types_evm.go b/badgerdb-sampler/types_evm.go index b19e0f1..3c693c9 100644 --- a/badgerdb-sampler/types_evm.go +++ b/badgerdb-sampler/types_evm.go @@ -3,6 +3,7 @@ package main // EVMDataInfo represents decoded EVM storage data. // See: _oasis-sdk/runtime-sdk/modules/evm/src/state.rs type EVMDataInfo struct { + RawError string `json:"raw_error,omitempty"` StorageType string `json:"storage_type"` // "code", "storage", "block_hash", "confidential_storage" Address string `json:"address,omitempty"` // H160 (20 bytes) - contract address (0x-prefixed hex) StorageSlot string `json:"storage_slot,omitempty"` // H256 (32 bytes) - storage slot (0x-prefixed hex, truncated) @@ -17,12 +18,7 @@ type EVMDataInfo struct { // See: _oasis-sdk/runtime-sdk/modules/evm/src/lib.rs:253-263 (Event::Log) // See: _oasis-sdk/client-sdk/go/modules/evm/types.go:56-61 (Event) type EVMEventInfo struct { - // Raw fields - RawDump string `json:"raw_dump,omitempty"` - RawSize int `json:"raw_size"` - RawError string `json:"raw_error,omitempty"` - - // Decoded fields + RawError string `json:"raw_error,omitempty"` Address string `json:"address"` // H160 contract address (0x-prefixed hex) TopicCount int `json:"topic_count"` // Number of topics (0-4) Topics []string `json:"topics,omitempty"` // H256 topics array (0x-prefixed hex) @@ -35,12 +31,7 @@ type EVMEventInfo struct { // EVMTxInputInfo represents decoded EVM transaction input artifacts. // See: _oasis-core/go/runtime/transaction/transaction.go:126-140 (inputArtifacts) type EVMTxInputInfo struct { - // Raw fields - RawDump string `json:"raw_dump,omitempty"` - RawSize int `json:"raw_size"` - RawError string `json:"raw_error,omitempty"` - - // Decoded fields + RawError string `json:"raw_error,omitempty"` TxHash string `json:"tx_hash"` // Transaction hash (0x-prefixed hex) BatchOrder uint32 `json:"batch_order"` // Order within batch Method string `json:"method,omitempty"` // SDK method (e.g., "evm.Call") @@ -49,12 +40,7 @@ type EVMTxInputInfo struct { // EVMTransactionInfo represents a decoded EVM transaction. type EVMTransactionInfo struct { - // Raw EVM transaction body bytes (CBOR) - RawDump string `json:"raw_dump,omitempty"` - RawSize int `json:"raw_size"` RawError string `json:"raw_error,omitempty"` - - // Decoded EVM transaction fields Type string `json:"type"` // "call" or "create" From string `json:"from,omitempty"` // 0x prefix To string `json:"to,omitempty"` // 0x prefix @@ -69,12 +55,7 @@ type EVMTransactionInfo struct { // EVMTxOutputInfo represents decoded EVM transaction output artifacts. // See: _oasis-core/go/runtime/transaction/transaction.go:142-150 (outputArtifacts) type EVMTxOutputInfo struct { - // Raw fields - RawDump string `json:"raw_dump,omitempty"` - RawSize int `json:"raw_size"` RawError string `json:"raw_error,omitempty"` - - // Decoded execution status fields SuccessExecution bool `json:"success"` ResultSize int `json:"result_size,omitempty"` ResultDump string `json:"result_dump,omitempty"` diff --git a/badgerdb-sampler/types_runtime.go b/badgerdb-sampler/types_runtime.go index 72880cb..f70bb2d 100644 --- a/badgerdb-sampler/types_runtime.go +++ b/badgerdb-sampler/types_runtime.go @@ -322,3 +322,69 @@ type cborRuntimeConsensusUndelegateDoneEvent struct { Shares []byte Amount RuntimeBaseUnits } + + +// ============================================================================= +// Runtime Module State Format Mappings +// ============================================================================= + +// RuntimeModuleStateFormat describes runtime module state value formats. +type RuntimeModuleStateFormat struct { + Format string // "cbor", "binary", "wasm" + Type string // Type name or description + Description string +} + +// runtimeModuleStateFormats maps module state keys to expected formats. +// Extracted from _oasis-sdk/runtime-sdk/modules/*/src/state.rs +var runtimeModuleStateFormats = map[string]map[byte]RuntimeModuleStateFormat{ + "evm": { + // _oasis-sdk/runtime-sdk/modules/evm/src/state.rs:10 + 0x01: {Format: "binary", Type: "Vec", Description: "contract code"}, + // _oasis-sdk/runtime-sdk/modules/evm/src/state.rs:12 + 0x02: {Format: "binary", Type: "H256", Description: "storage slot"}, + // _oasis-sdk/runtime-sdk/modules/evm/src/state.rs:14 + 0x03: {Format: "binary", Type: "H256", Description: "block hash"}, + // _oasis-sdk/runtime-sdk/modules/evm/src/state.rs:17 + 0x04: {Format: "binary", Type: "ConfidentialValue", Description: "confidential storage"}, + }, + "accounts": { + // _oasis-sdk/runtime-sdk/modules/accounts/src/state.rs + 0x01: {Format: "cbor", Type: "types.AccountInfo", Description: "account"}, + 0x02: {Format: "cbor", Type: "types.AccountBalances", Description: "balances"}, + 0x03: {Format: "cbor", Type: "Quantity", Description: "total supply"}, + }, + "contracts": { + // _oasis-sdk/runtime-sdk/modules/contracts/src/state.rs + 0x01: {Format: "cbor", Type: "u64", Description: "next code ID"}, + 0x02: {Format: "cbor", Type: "u64", Description: "next instance ID"}, + 0x03: {Format: "cbor", Type: "types.Code", Description: "code info"}, + 0x04: {Format: "cbor", Type: "types.Instance", Description: "instance info"}, + 0x05: {Format: "cbor", Type: "store.Value", Description: "instance state"}, + 0xFF: {Format: "wasm", Type: "Vec", Description: "WASM bytecode"}, + }, + "core": { + // _oasis-sdk/runtime-sdk/src/modules/core/state.rs + 0x01: {Format: "cbor", Type: "types.Metadata", Description: "metadata"}, + 0x02: {Format: "cbor", Type: "MessageHandlers", Description: "message handlers"}, + 0x03: {Format: "cbor", Type: "EpochTime", Description: "last epoch"}, + 0x04: {Format: "cbor", Type: "Quantity", Description: "min gas price"}, + }, + "consensus_accounts": { + // _oasis-sdk/runtime-sdk/modules/consensus_accounts/src/state.rs + 0x01: {Format: "cbor", Type: "types.Delegation", Description: "delegations"}, + 0x02: {Format: "cbor", Type: "types.UndelegationReceipt", Description: "undelegations"}, + 0x03: {Format: "cbor", Type: "QueueEntry", Description: "undelegation queue"}, + 0x04: {Format: "cbor", Type: "types.Receipt", Description: "receipts"}, + }, +} + +// GetRuntimeModuleStateFormat returns format for a module state key. +func GetRuntimeModuleStateFormat(module string, subPrefix byte) (RuntimeModuleStateFormat, bool) { + if moduleFmts, exists := runtimeModuleStateFormats[module]; exists { + if format, exists := moduleFmts[subPrefix]; exists { + return format, true + } + } + return RuntimeModuleStateFormat{}, false +} From 5ff6acc758755a5e9d76d2f687e785b6f14d4766 Mon Sep 17 00:00:00 2001 From: gw0 Date: Wed, 3 Dec 2025 13:03:37 +0100 Subject: [PATCH 19/22] badgerdb-sampler: Improve consistency and error details --- badgerdb-sampler/decode_consensus.go | 15 ++++--- badgerdb-sampler/decode_runtime.go | 65 +++++++++++++++++++++++----- badgerdb-sampler/main.go | 40 ++++++++--------- badgerdb-sampler/utils.go | 20 +++++++++ 4 files changed, 101 insertions(+), 39 deletions(-) diff --git a/badgerdb-sampler/decode_consensus.go b/badgerdb-sampler/decode_consensus.go index 969ac55..c47575e 100644 --- a/badgerdb-sampler/decode_consensus.go +++ b/badgerdb-sampler/decode_consensus.go @@ -435,7 +435,8 @@ func decodeConsensusMkvsValue(keyType string, value []byte) *ConsensusMkvsValueI data = data[2:] if len(data) < keySize { - info.RawError = fmt.Sprintf("key truncated") + info.RawError = fmt.Sprintf("key truncated (expected %s, got %s)", + formatApproxSize(keySize), formatApproxSize(len(data))) return info } @@ -478,7 +479,8 @@ func decodeConsensusMkvsValue(keyType string, value []byte) *ConsensusMkvsValueI leaf.ValueSize = valueSize if len(data) < valueSize { - info.RawError = fmt.Sprintf("value truncated") + info.RawError = fmt.Sprintf("value truncated (expected %s, got %s)", + formatApproxSize(valueSize), formatApproxSize(len(data))) info.Leaf = leaf return info } @@ -558,7 +560,8 @@ func decodeConsensusMkvsValue(keyType string, value []byte) *ConsensusMkvsValueI labelBytes := (int(labelBits) + 7) / 8 if len(data) < labelBytes+1 { - info.RawError = "label truncated" + info.RawError = fmt.Sprintf("label truncated (expected %s, got %s)", + formatApproxSize(labelBytes+1), formatApproxSize(len(data))) info.Internal = &ConsensusMkvsInternalInfo{LabelBits: labelBits} return info } @@ -589,7 +592,8 @@ func decodeConsensusMkvsValue(keyType string, value []byte) *ConsensusMkvsValueI keyLen := binary.LittleEndian.Uint16(data[0:2]) data = data[2:] if len(data) < int(keyLen)+4 { - info.RawError = "embedded leaf truncated" + info.RawError = fmt.Sprintf("embedded leaf truncated (expected %s, got %s)", + formatApproxSize(int(keyLen)+4), formatApproxSize(len(data))) info.Internal = internal return info } @@ -597,7 +601,8 @@ func decodeConsensusMkvsValue(keyType string, value []byte) *ConsensusMkvsValueI valueLen := binary.LittleEndian.Uint32(data[0:4]) data = data[4:] if len(data) < int(valueLen) { - info.RawError = "embedded leaf value truncated" + info.RawError = fmt.Sprintf("embedded leaf value truncated (expected %s, got %s)", + formatApproxSize(int(valueLen)), formatApproxSize(len(data))) info.Internal = internal return info } diff --git a/badgerdb-sampler/decode_runtime.go b/badgerdb-sampler/decode_runtime.go index 3bb2e7f..734b7c6 100644 --- a/badgerdb-sampler/decode_runtime.go +++ b/badgerdb-sampler/decode_runtime.go @@ -97,7 +97,8 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) *RuntimeMkvsValueInfo data = data[2:] if len(data) < keySize { - info.RawError = fmt.Sprintf("key truncated (expected %d bytes)", keySize) + info.RawError = fmt.Sprintf("key truncated (expected %s, got %s)", + formatApproxSize(keySize), formatApproxSize(len(data))) return info } @@ -123,7 +124,8 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) *RuntimeMkvsValueInfo data = data[4:] if len(data) < valueSize { - info.RawError = fmt.Sprintf("value truncated (expected %d bytes)", valueSize) + info.RawError = fmt.Sprintf("value truncated (expected %s, got %s)", + formatApproxSize(valueSize), formatApproxSize(len(data))) info.Leaf = leaf return info } @@ -155,7 +157,8 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) *RuntimeMkvsValueInfo labelBytes := (int(labelBits) + 7) / 8 if len(data) < labelBytes+1 { - info.RawError = "label truncated" + info.RawError = fmt.Sprintf("label truncated (expected %s, got %s)", + formatApproxSize(labelBytes+1), formatApproxSize(len(data))) info.Internal = &RuntimeMkvsInternalInfo{LabelBits: labelBits} return info } @@ -163,19 +166,57 @@ func decodeRuntimeMkvsValue(keyType string, value []byte) *RuntimeMkvsValueInfo data = data[labelBytes:] // skip label internal := &RuntimeMkvsInternalInfo{LabelBits: labelBits} - hasLeaf := data[0] == 0x00 - internal.HasLeaf = hasLeaf - if !hasLeaf { + // Check for embedded leaf node or nil marker + if len(data) < 1 { + info.RawError = "missing leaf/nil marker" + info.Internal = internal + return info + } + + if data[0] == 0x02 { // NilNode marker - no embedded leaf + internal.HasLeaf = false data = data[1:] // skip nil marker - // Extract child hashes - if len(data) >= 32 { - internal.LeftHash = formatRawValue(data[:32], TruncateHashLen) - data = data[32:] + } else if data[0] == 0x00 { // LeafNode prefix - embedded leaf present + internal.HasLeaf = true + // Skip embedded leaf: prefix(1) + keyLen(2) + key + valueLen(4) + value + data = data[1:] // skip prefix + if len(data) < 2 { + info.RawError = "embedded leaf key length missing" + info.Internal = internal + return info + } + keyLen := binary.LittleEndian.Uint16(data[0:2]) + data = data[2:] + if len(data) < int(keyLen)+4 { + info.RawError = fmt.Sprintf("embedded leaf truncated (expected %s, got %s)", + formatApproxSize(int(keyLen)+4), formatApproxSize(len(data))) + info.Internal = internal + return info } - if len(data) >= 32 { - internal.RightHash = formatRawValue(data[:32], TruncateHashLen) + data = data[keyLen:] // skip key + valueLen := binary.LittleEndian.Uint32(data[0:4]) + data = data[4:] + if len(data) < int(valueLen) { + info.RawError = fmt.Sprintf("embedded leaf value truncated (expected %s, got %s)", + formatApproxSize(int(valueLen)), formatApproxSize(len(data))) + info.Internal = internal + return info } + data = data[valueLen:] // skip value + } else { + info.RawError = fmt.Sprintf("unexpected marker 0x%02x", data[0]) + info.Internal = internal + return info + } + + // Read left and right hashes + if len(data) >= 32 { + internal.LeftHash = formatRawValue(data[:32], TruncateHashLen) + data = data[32:] + } + if len(data) >= 32 { + internal.RightHash = formatRawValue(data[:32], TruncateHashLen) } info.Internal = internal diff --git a/badgerdb-sampler/main.go b/badgerdb-sampler/main.go index e7cd52d..01c8f52 100644 --- a/badgerdb-sampler/main.go +++ b/badgerdb-sampler/main.go @@ -90,22 +90,6 @@ func main() { log.Fatalf("Error collecting samples: %v", err) } - // Extract and count errors from all samples - fmt.Fprintf(os.Stderr, "Extracting errors from samples...\n") - for _, sample := range stats.Samples { - // Extract errors from key - keyErrors := extractErrors(sample.Key, "key", 10) - for _, errMsg := range keyErrors { - stats.ErrorCounts[errMsg]++ - } - - // Extract errors from value - valueErrors := extractErrors(sample.Value, "value", 10) - for _, errMsg := range valueErrors { - stats.ErrorCounts[errMsg]++ - } - } - // Print results fmt.Printf("Results:\n") output, err := json.MarshalIndent(stats, "", " ") @@ -227,7 +211,7 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { if err != nil { sample.Value = &ConsensusBlockstoreValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err), RawSize: valSize} } else if len(val) != valSize { - sample.Value = &ConsensusBlockstoreValueInfo{RawError: fmt.Sprintf("value size mismatch (got: %d)", len(val)), RawSize: valSize} + sample.Value = &ConsensusBlockstoreValueInfo{RawError: fmt.Sprintf("value size mismatch (expected: %s, got: %s)", formatApproxSize(valSize), formatApproxSize(len(val))), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} } else { sample.Value = decodeConsensusBlockstoreValue(sample.KeyType, val) sample.Timestamp = sample.Value.(*ConsensusBlockstoreValueInfo).Timestamp @@ -238,7 +222,7 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { if err != nil { sample.Value = &ConsensusEvidenceValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err), RawSize: valSize} } else if len(val) != valSize { - sample.Value = &ConsensusEvidenceValueInfo{RawError: fmt.Sprintf("value size mismatch (got: %d)", len(val)), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} + sample.Value = &ConsensusEvidenceValueInfo{RawError: fmt.Sprintf("value size mismatch (expected: %s, got: %s)", formatApproxSize(valSize), formatApproxSize(len(val))), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} } else { sample.Value = decodeConsensusEvidenceValue(sample.KeyType, val) } @@ -248,7 +232,7 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { if err != nil { sample.Value = &ConsensusMkvsValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err), RawSize: valSize} } else if len(val) != valSize { - sample.Value = &ConsensusMkvsValueInfo{RawError: fmt.Sprintf("value size mismatch (got: %d)", len(val)), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} + sample.Value = &ConsensusMkvsValueInfo{RawError: fmt.Sprintf("value size mismatch (expected: %s, got: %s)", formatApproxSize(valSize), formatApproxSize(len(val))), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} } else { sample.Value = decodeConsensusMkvsValue(sample.KeyType, val) } @@ -258,7 +242,7 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { if err != nil { sample.Value = &ConsensusStateValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err), RawSize: valSize} } else if len(val) != valSize { - sample.Value = &ConsensusStateValueInfo{RawError: fmt.Sprintf("value size mismatch (got: %d)", len(val)), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} + sample.Value = &ConsensusStateValueInfo{RawError: fmt.Sprintf("value size mismatch (expected: %s, got: %s)", formatApproxSize(valSize), formatApproxSize(len(val))), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} } else { sample.Value = decodeConsensusStateValue(sample.KeyType, val) } @@ -268,7 +252,7 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { if err != nil { sample.Value = &RuntimeMkvsValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err), RawSize: valSize} } else if len(val) != valSize { - sample.Value = &RuntimeMkvsValueInfo{RawError: fmt.Sprintf("value size mismatch (got: %d)", len(val)), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} + sample.Value = &RuntimeMkvsValueInfo{RawError: fmt.Sprintf("value size mismatch (expected: %s, got: %s)", formatApproxSize(valSize), formatApproxSize(len(val))), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} } else { sample.Value = decodeRuntimeMkvsValue(sample.KeyType, val) } @@ -278,7 +262,7 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { if err != nil { sample.Value = &RuntimeHistoryValueInfo{RawError: fmt.Sprintf("failed to read value: %v", err), RawSize: valSize} } else if len(val) != valSize { - sample.Value = &RuntimeHistoryValueInfo{RawError: fmt.Sprintf("value size mismatch (got: %d)", len(val)), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} + sample.Value = &RuntimeHistoryValueInfo{RawError: fmt.Sprintf("value size mismatch (expected: %s, got: %s)", formatApproxSize(valSize), formatApproxSize(len(val))), RawSize: valSize, RawDump: formatRawValue(val, TruncateLongLen)} } else { sample.Value = decodeRuntimeHistoryValue(sample.KeyType, val) } @@ -286,6 +270,18 @@ func collectSamples(db *DB, stats *DBStats, maxSamples int) error { log.Fatalf("Unsupported database type: %s", stats.DatabaseType) } + // Extract and count errors from key + keyErrors := extractErrors(sample.Key, "key", 10) + for _, errMsg := range keyErrors { + stats.ErrorCounts[errMsg]++ + } + + // Extract and count errors from value + valueErrors := extractErrors(sample.Value, "value", 10) + for _, errMsg := range valueErrors { + stats.ErrorCounts[errMsg]++ + } + stats.Samples = append(stats.Samples, sample) stats.KeyTypeCounts[sample.KeyType]++ sampleCount++ diff --git a/badgerdb-sampler/utils.go b/badgerdb-sampler/utils.go index 07439f2..f8f238a 100644 --- a/badgerdb-sampler/utils.go +++ b/badgerdb-sampler/utils.go @@ -435,3 +435,23 @@ func toSnakeCase(s string) string { } return result.String() } + +// formatApproxSize returns approximate size string for error messages. +// Rounds to first significant digit: 23→~20, 234→~200, 2345→~2K, 23456→~20K +func formatApproxSize(size int) string { + if size < 10 { + return fmt.Sprintf("~%d", size) + } else if size < 100 { + return fmt.Sprintf("~%d", (size/10)*10) + } else if size < 1000 { + return fmt.Sprintf("~%d", (size/100)*100) + } else if size < 10000 { + return fmt.Sprintf("~%dK", size/1000) + } else if size < 100000 { + return fmt.Sprintf("~%dK", (size/10000)*10) + } else if size < 1000000 { + return fmt.Sprintf("~%dK", (size/100000)*100) + } else { + return fmt.Sprintf("~%dM", size/1000000) + } +} From 27031d6ee9ff13f34a6d97cff0d1fb25d92cacd5 Mon Sep 17 00:00:00 2001 From: gw0 Date: Wed, 3 Dec 2025 13:05:19 +0100 Subject: [PATCH 20/22] badgerdb-sampler: Improve building for multiple BadgerDB versions --- badgerdb-sampler/Makefile | 19 ++-- badgerdb-sampler/go.mod | 29 +++-- badgerdb-sampler/go.sum | 60 +++++++---- badgerdb-sampler/go_v2.mod | 34 ++++-- badgerdb-sampler/go_v2.sum | 143 ++++++++++++++++++++++--- badgerdb-sampler/go_v3.mod | 29 +++-- badgerdb-sampler/go_v3.sum | 60 +++++++---- badgerdb-sampler/go_v4.mod | 26 ++++- badgerdb-sampler/go_v4.sum | 210 +++++++++++++++++++++++++++++++++++++ 9 files changed, 519 insertions(+), 91 deletions(-) create mode 100644 badgerdb-sampler/go_v4.sum diff --git a/badgerdb-sampler/Makefile b/badgerdb-sampler/Makefile index a9642ca..04f28b7 100644 --- a/badgerdb-sampler/Makefile +++ b/badgerdb-sampler/Makefile @@ -1,6 +1,6 @@ # Makefile for badgerdb-sampler -.PHONY: all build-all build-v2 build-v3 build-v4 clean +.PHONY: all build-all build-v2 build-v3 build-v4 clean tidy-all # Default target all: build-all @@ -11,22 +11,19 @@ build-all: build-v2 build-v3 build-v4 # Build BadgerDB v2 version build-v2: @echo "Building badgerdb-sampler-v2..." - @cp go_v2.mod go.mod && cp go_v2.sum go.sum 2>/dev/null || true - go build -tags badgerv2 -o bin/badgerdb-sampler-v2 + go build -modfile=go_v2.mod -tags badgerv2 -o bin/badgerdb-sampler-v2 @echo "✓ Built: bin/badgerdb-sampler-v2" # Build BadgerDB v3 version build-v3: @echo "Building badgerdb-sampler-v3..." - @cp go_v3.mod go.mod && cp go_v3.sum go.sum 2>/dev/null || true - go build -tags badgerv3 -o bin/badgerdb-sampler-v3 + go build -modfile=go_v3.mod -tags badgerv3 -o bin/badgerdb-sampler-v3 @echo "✓ Built: bin/badgerdb-sampler-v3" # Build BadgerDB v4 version build-v4: @echo "Building badgerdb-sampler-v4..." - @cp go_v4.mod go.mod && cp go_v4.sum go.sum 2>/dev/null || true - go build -tags badgerv4 -o bin/badgerdb-sampler-v4 + go build -modfile=go_v4.mod -tags badgerv4 -o bin/badgerdb-sampler-v4 @echo "✓ Built: bin/badgerdb-sampler-v4" # Clean build artifacts @@ -36,3 +33,11 @@ clean: @echo "Cleaning outputs/..." rm -rf outputs/* @echo "✓ Cleaned" + +# Tidy all module files +tidy-all: + @echo "Tidying all module files..." + GOFLAGS="-tags=badgerv2" go mod tidy -modfile=go_v2.mod + GOFLAGS="-tags=badgerv3" go mod tidy -modfile=go_v3.mod + GOFLAGS="-tags=badgerv4" go mod tidy -modfile=go_v4.mod + @echo "✓ All module files tidied" diff --git a/badgerdb-sampler/go.mod b/badgerdb-sampler/go.mod index 1db1dea..dcbe6cc 100644 --- a/badgerdb-sampler/go.mod +++ b/badgerdb-sampler/go.mod @@ -1,29 +1,36 @@ module github.com/oasisprotocol/badgerdb-sampler -go 1.25.4 +go 1.23.0 require ( - github.com/dgraph-io/badger/v3 v3.2103.5 + // BadgerDB dependency filled by go_*.mod + github.com/fxamacker/cbor/v2 v2.9.0 github.com/gogo/protobuf v1.3.2 - github.com/tendermint/tendermint v0.34.21 ) require ( github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect + github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.3 // indirect - github.com/google/flatbuffers v1.12.1 // indirect - github.com/klauspost/compress v1.15.9 // indirect + github.com/google/flatbuffers v25.2.10+incompatible // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/x448/float16 v0.8.4 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/net v0.0.0-20220726230323-06994584191e // indirect - golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect - google.golang.org/protobuf v1.28.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sys v0.34.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect ) diff --git a/badgerdb-sampler/go.sum b/badgerdb-sampler/go.sum index 89410b4..982e58c 100644 --- a/badgerdb-sampler/go.sum +++ b/badgerdb-sampler/go.sum @@ -7,8 +7,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -18,14 +18,23 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= +github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/badger/v4 v4.8.0 h1:JYph1ChBijCw8SLeybvPINizbDKWZ5n/GYbz2yhN/bs= +github.com/dgraph-io/badger/v4 v4.8.0/go.mod h1:U6on6e8k/RTbUWxqKR0MvugJuVmkxSNc79ap4917h4w= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM= +github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -33,6 +42,11 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -57,8 +71,9 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= +github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -67,16 +82,16 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -84,6 +99,7 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -103,10 +119,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/tendermint/tendermint v0.34.21 h1:UiGGnBFHVrZhoQVQ7EfwSOLuCtarqCSsRf8VrklqB7s= -github.com/tendermint/tendermint v0.34.21/go.mod h1:XDvfg6U7grcFTDx7VkzxnhazQ/bspGJAn4DZ6DcLLjQ= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= @@ -116,6 +130,14 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -135,8 +157,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20220726230323-06994584191e h1:wOQNKh1uuDGRnmgF0jDxh7ctgGy/3P4rYWQRVJD4/Yg= -golang.org/x/net v0.0.0-20220726230323-06994584191e/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -149,9 +171,11 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -189,8 +213,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/badgerdb-sampler/go_v2.mod b/badgerdb-sampler/go_v2.mod index ae9b17b..a3c0be1 100644 --- a/badgerdb-sampler/go_v2.mod +++ b/badgerdb-sampler/go_v2.mod @@ -1,25 +1,39 @@ module github.com/oasisprotocol/badgerdb-sampler -go 1.21 +go 1.23.0 require ( github.com/dgraph-io/badger/v2 v2.2007.2 + github.com/dgraph-io/badger/v3 v3.2103.5 + github.com/dgraph-io/badger/v4 v4.8.0 + github.com/fxamacker/cbor/v2 v2.9.0 github.com/gogo/protobuf v1.3.2 - github.com/tendermint/tendermint v0.34.21 ) require ( github.com/DataDog/zstd v1.4.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect - github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect + github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/golang/snappy v0.0.1 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/flatbuffers v25.2.10+incompatible // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/net v0.0.0-20220726230323-06994584191e // indirect - golang.org/x/sys v0.0.0-20220727055044-e65921a090b8 // indirect - google.golang.org/protobuf v1.28.0 // indirect + go.opencensus.io v0.23.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sys v0.34.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect ) diff --git a/badgerdb-sampler/go_v2.sum b/badgerdb-sampler/go_v2.sum index 6eaf54d..9bfb170 100644 --- a/badgerdb-sampler/go_v2.sum +++ b/badgerdb-sampler/go_v2.sum @@ -1,11 +1,18 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -15,30 +22,79 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= +github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= +github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/badger/v4 v4.8.0 h1:JYph1ChBijCw8SLeybvPINizbDKWZ5n/GYbz2yhN/bs= +github.com/dgraph-io/badger/v4 v4.8.0/go.mod h1:U6on6e8k/RTbUWxqKR0MvugJuVmkxSNc79ap4917h4w= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM= +github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= +github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -51,6 +107,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -64,41 +121,71 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/tendermint/tendermint v0.34.21 h1:UiGGnBFHVrZhoQVQ7EfwSOLuCtarqCSsRf8VrklqB7s= -github.com/tendermint/tendermint v0.34.21/go.mod h1:XDvfg6U7grcFTDx7VkzxnhazQ/bspGJAn4DZ6DcLLjQ= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20220726230323-06994584191e h1:wOQNKh1uuDGRnmgF0jDxh7ctgGy/3P4rYWQRVJD4/Yg= -golang.org/x/net v0.0.0-20220726230323-06994584191e/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220727055044-e65921a090b8 h1:dyU22nBWzrmTQxtNrr4dzVOvaw35nUYE279vF9UmsI8= -golang.org/x/sys v0.0.0-20220727055044-e65921a090b8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -106,12 +193,36 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/badgerdb-sampler/go_v3.mod b/badgerdb-sampler/go_v3.mod index 1db1dea..6d1777f 100644 --- a/badgerdb-sampler/go_v3.mod +++ b/badgerdb-sampler/go_v3.mod @@ -1,29 +1,38 @@ module github.com/oasisprotocol/badgerdb-sampler -go 1.25.4 +go 1.23.0 require ( + github.com/dgraph-io/badger/v2 v2.2007.4 github.com/dgraph-io/badger/v3 v3.2103.5 + github.com/dgraph-io/badger/v4 v4.8.0 + github.com/fxamacker/cbor/v2 v2.9.0 github.com/gogo/protobuf v1.3.2 - github.com/tendermint/tendermint v0.34.21 ) require ( github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect + github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.3 // indirect - github.com/google/flatbuffers v1.12.1 // indirect - github.com/klauspost/compress v1.15.9 // indirect + github.com/google/flatbuffers v25.2.10+incompatible // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/x448/float16 v0.8.4 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/net v0.0.0-20220726230323-06994584191e // indirect - golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect - google.golang.org/protobuf v1.28.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sys v0.34.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect ) diff --git a/badgerdb-sampler/go_v3.sum b/badgerdb-sampler/go_v3.sum index 89410b4..982e58c 100644 --- a/badgerdb-sampler/go_v3.sum +++ b/badgerdb-sampler/go_v3.sum @@ -7,8 +7,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -18,14 +18,23 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= +github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/badger/v4 v4.8.0 h1:JYph1ChBijCw8SLeybvPINizbDKWZ5n/GYbz2yhN/bs= +github.com/dgraph-io/badger/v4 v4.8.0/go.mod h1:U6on6e8k/RTbUWxqKR0MvugJuVmkxSNc79ap4917h4w= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM= +github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -33,6 +42,11 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -57,8 +71,9 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= +github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -67,16 +82,16 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -84,6 +99,7 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -103,10 +119,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/tendermint/tendermint v0.34.21 h1:UiGGnBFHVrZhoQVQ7EfwSOLuCtarqCSsRf8VrklqB7s= -github.com/tendermint/tendermint v0.34.21/go.mod h1:XDvfg6U7grcFTDx7VkzxnhazQ/bspGJAn4DZ6DcLLjQ= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= @@ -116,6 +130,14 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -135,8 +157,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20220726230323-06994584191e h1:wOQNKh1uuDGRnmgF0jDxh7ctgGy/3P4rYWQRVJD4/Yg= -golang.org/x/net v0.0.0-20220726230323-06994584191e/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -149,9 +171,11 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -189,8 +213,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/badgerdb-sampler/go_v4.mod b/badgerdb-sampler/go_v4.mod index f9332c6..7b2272b 100644 --- a/badgerdb-sampler/go_v4.mod +++ b/badgerdb-sampler/go_v4.mod @@ -3,9 +3,33 @@ module github.com/oasisprotocol/badgerdb-sampler go 1.25.4 require ( + github.com/dgraph-io/badger/v2 v2.2007.4 + github.com/dgraph-io/badger/v3 v3.2103.5 github.com/dgraph-io/badger/v4 v4.2.0 + github.com/fxamacker/cbor/v2 v2.9.0 github.com/gogo/protobuf v1.3.2 - github.com/tendermint/tendermint v0.34.21 +) + +require ( + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/golang/glog v1.0.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/flatbuffers v1.12.1 // indirect + github.com/google/go-cmp v0.5.8 // indirect + github.com/klauspost/compress v1.15.9 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/stretchr/testify v1.8.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.opencensus.io v0.23.0 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.5.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect ) // Note: Dependencies will be filled in when v4 support is implemented diff --git a/badgerdb-sampler/go_v4.sum b/badgerdb-sampler/go_v4.sum new file mode 100644 index 0000000..49ec425 --- /dev/null +++ b/badgerdb-sampler/go_v4.sum @@ -0,0 +1,210 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= +github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= +github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= +github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From a59c24b1f1ef5e836fa8a14cacfbd85614c64617 Mon Sep 17 00:00:00 2001 From: gw0 Date: Wed, 3 Dec 2025 13:05:45 +0100 Subject: [PATCH 21/22] badgerdb-sampler: Update README and improve helper scripts --- badgerdb-sampler/README.md | 113 ++++++++++--- badgerdb-sampler/scripts/mount-snapshots.sh | 30 ---- .../scripts/run-mainnet-samplers.sh | 137 ++++++++++------ .../scripts/run-testnet-samplers.sh | 152 +++++++++++------- badgerdb-sampler/scripts/snapshots-extract.sh | 64 ++++++++ badgerdb-sampler/scripts/umount-snapshots.sh | 20 --- 6 files changed, 343 insertions(+), 173 deletions(-) delete mode 100755 badgerdb-sampler/scripts/mount-snapshots.sh create mode 100755 badgerdb-sampler/scripts/snapshots-extract.sh delete mode 100755 badgerdb-sampler/scripts/umount-snapshots.sh diff --git a/badgerdb-sampler/README.md b/badgerdb-sampler/README.md index 4b8c163..17cf9f3 100644 --- a/badgerdb-sampler/README.md +++ b/badgerdb-sampler/README.md @@ -1,12 +1,35 @@ # BadgerDB Sampler -A tool for sampling BadgerDB v2, v3, and v4 databases used by Oasis nodes. +This **badgerdb-sampler** tool implements a comprehensive data extraction and decoding logic for BadgerDB databases used in Oasis node snapshots. This tool can extract and decode most data structures from consensus and runtime databases, even EVM data. It handles multiple BadgerDB versions (v2-v4) and data representations use by Oasis nodes (v20.x-v25.x). The decoding logic handles various data formats and reports decoding issues. + +## Features + +- **Multi-version BadgerDB Support**: Compatible with BadgerDB v2, v3, and v4 +- **Comprehensive Decoding**: Handles consensus state, runtime state, and EVM-specific data +- **Error Tracking**: Collects and reports decoding errors with detailed error counts +- **Statistics Generation**: Provides key type distributions, database size, and sample counts +- **Module-aware Parsing**: Recognizes and routes data to appropriate decoders (evm, accounts, contracts, core) +- **EVM Event Decoding**: Includes event signature database for human-readable EVM event names +- **FUSE Filesystem Support**: Works with databases on FUSE mounts using intelligent fallback strategies +- **Read-only Access**: Can analyze databases currently in use by nodes via `BypassLockGuard` + +## Supported Database Types + +- `consensus-blockstore` - Block metadata and commit info +- `consensus-evidence` - Byzantine validator evidence +- `consensus-mkvs` - Consensus state Merkle tree +- `consensus-state` - Tendermint/CometBFT consensus state +- `runtime-mkvs` - Runtime state Merkle tree (includes EVM storage) +- `runtime-history` - Runtime block history with CBOR-encoded data (includes EVM events/transactions) ## Building ```bash -# Build all versions +# Build all versions (recommended) make build-all + +# Clean build artifacts +make clean ``` ## Usage @@ -16,29 +39,83 @@ make build-all - Go 1.21 or higher - BadgerDB databases from Oasis nodes (see https://snapshots.oasis.io/) -### Run +### Command Syntax ```bash -# Same interface for all versions -./badgerdb-sampler-v2 [output-subdir] [output-prefix] [max-samples] -./badgerdb-sampler-v3 [output-subdir] [output-prefix] [max-samples] +./bin/badgerdb-sampler-v{2,3,4} [output-json] [max-samples] ``` -Database types (version suffixes are optional): - -- `consensus-blockstore` (or `consensus-blockstore-v2`, `consensus-blockstore-v3`) -- `consensus-evidence` -- `consensus-mkvs` -- `consensus-state` -- `runtime-mkvs` -- `runtime-history` +**Parameters:** +- `database-type`: One of the supported database types (see above) +- `path-to-db`: Path to the BadgerDB database directory +- `output-json`: Optional path to save JSON results (default: stdout only) +- `max-samples`: Optional maximum number of samples to collect (default: 1000) ### Examples ```bash -# Analyze v2 consensus blockstore -./badgerdb-sampler-v2 consensus-blockstore /path/to/blockstore.badger.db ./outputs/testnet-20220303/ +# Analyze v2 consensus blockstore (default 1000 samples, stdout only) +./bin/badgerdb-sampler-v2 consensus-blockstore /path/to/blockstore.badger.db + +# Analyze v3 runtime MKVS with JSON output +./bin/badgerdb-sampler-v3 runtime-mkvs /path/to/mkvs_storage.badger.db ./outputs/testnet-20220303/emerald-runtime-mkvs.json + +# Analyze runtime history with custom sample limit +./bin/badgerdb-sampler-v3 runtime-history /path/to/history.badger.db ./outputs/runtime-history.json 500 + +# Analyze currently running node's database (read-only) +./bin/badgerdb-sampler-v3 consensus-state /var/lib/oasis/node/consensus/state.badger.db ./outputs/consensus-state.json +``` + +## Output Format + +The tool outputs JSON with comprehensive statistics and samples: -# Analyze v3 runtime MKVS -./badgerdb-sampler-v3 runtime-mkvs /path/to/mkvs_storage.badger.db ./outputs/testnet-20220303/emerald- 500 +```json +{ + "database_path": "/path/to/mkvs_storage.badger.db", + "database_type": "runtime-mkvs", + "badgerdb_version": "v3", + "database_size_bytes": 1234567890, + "sample_count": 1000, + "key_type_counts": { + "mkvs:node": 850, + "mkvs:root": 150 + }, + "error_counts": { + "failed to decode CBOR value: unexpected EOF": 5, + "unknown module prefix": 2 + }, + "samples": [ + { + "key_type": "mkvs:node", + "key": { /* decoded key structure */ }, + "value": { /* decoded value structure */ }, + } + ] +} ``` + +**Key Fields:** +- `key_type_counts`: Distribution of different key types in the database +- `error_counts`: Aggregated decoding errors across all samples +- `samples`: Array of individual key-value pairs with full decoding details + +## Architecture + +The tool uses a three-layer design: + +1. **Database Access** (`db_v*.go`, `db_common.go`) + - Version-specific BadgerDB initialization via build tags + - FUSE workaround with temp directory and symlinks + - Read-only mode with lock bypass for in-use databases + +2. **Decoding Logic** (`decode_*.go`) + - `decode_consensus.go`: Tendermint protobuf parsing + - `decode_runtime.go`: CBOR/MKVS parsing with module routing + - `decode_evm.go`: EVM-specific parsing (contracts, events, transactions) + +3. **Type System** (`types_*.go`) + - Separated deserialization and output types + - EVM-specific output structures + - Event signature database for human-readable names diff --git a/badgerdb-sampler/scripts/mount-snapshots.sh b/badgerdb-sampler/scripts/mount-snapshots.sh deleted file mode 100755 index e587a46..0000000 --- a/badgerdb-sampler/scripts/mount-snapshots.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Script to mount .tar.zst snapshot archives without extracting -# -# ./mount-snapshots.sh testnet -# -# Manually: -# ./ratarmount -o allow_other,uid=1000,gid=1000 -w "testnet/20220303-20231012/cipher_testnet.overlay" "testnet/20220303-20231012/cipher_testnet.tar.zst" "testnet/20220303-20231012/cipher_testnet.mount" -# fusermount -u "testnet/20220303-20231012/cipher_testnet.mount" && rm -rf "testnet/20220303-20231012/cipher_testnet.mount" "testnet/20220303-20231012/cipher_testnet.overlay" -# -set -euo pipefail - -cd "$(dirname "$0")" -search_dir="${1:-.}" - -find "$search_dir" -name "*.tar.zst" -type f | while read -r archive; do - basename="${archive%.tar.zst}" - mountpoint="${basename}.mount" - overlay="${basename}.overlay" - - if mountpoint -q "$mountpoint" 2>/dev/null; then - echo "Already mounted: $mountpoint" - continue - fi - - echo "Mounting: $archive -> $mountpoint" - mkdir -p "$mountpoint" - ./ratarmount -o allow_other,uid=1000,gid=1000 -w "$overlay" "$archive" "$mountpoint" -done - -echo "Done!" diff --git a/badgerdb-sampler/scripts/run-mainnet-samplers.sh b/badgerdb-sampler/scripts/run-mainnet-samplers.sh index 217eab8..3998b29 100755 --- a/badgerdb-sampler/scripts/run-mainnet-samplers.sh +++ b/badgerdb-sampler/scripts/run-mainnet-samplers.sh @@ -1,68 +1,109 @@ #!/bin/bash -# Script to run all badgerdb-samplers on Mainnet +# Script to run all badgerdb-samplers on Mainnet snapshots set -euo pipefail SAMPLER_V2="./bin/badgerdb-sampler-v2" SAMPLER_V3="./bin/badgerdb-sampler-v3" -OUTPUT="./outputs" +pre_run() { + mkdir -p "${OUTPUT_DIR}" + "$(dirname "${0}")"/snapshots-extract.sh --extract "${DATA_DIR%.dir}.tar.zst" + echo "Processing: ${DATA_DIR}..." +} +post_run() { + "$(dirname "${0}")"/snapshots-extract.sh --clean "${DATA_DIR%.dir}.tar.zst" + echo "Processing done." + echo +} -mkdir -p "$OUTPUT" -# $OUTPUT/mainnet/20201001-20201118/ (BadgerDB v2) -mkdir -p "$OUTPUT/mainnet-20201001-20201118" +# mainnet-20201001-20201118 (BadgerDB v2) ( -$SAMPLER_V2 consensus-blockstore-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20201001-20201118/consensus-blockstore-v2.json > $OUTPUT/mainnet-20201001-20201118/consensus-blockstore-v2.log 2>&1 || true -$SAMPLER_V2 consensus-evidence-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20201001-20201118/consensus-evidence-v2.json > $OUTPUT/mainnet-20201001-20201118/consensus-evidence-v2.log 2>&1 || true -$SAMPLER_V2 consensus-state-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20201001-20201118/consensus-state-v2.json > $OUTPUT/mainnet-20201001-20201118/consensus-state-v2.log 2>&1 || true -$SAMPLER_V2 consensus-mkvs-v2 /snapshots/mainnet/20201001-20201118/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20201001-20201118/consensus-mkvs-v2.json > $OUTPUT/mainnet-20201001-20201118/consensus-mkvs-v2.log 2>&1 || true -) & +DATA_DIR="/snapshots/mainnet/20201001-20201118/consensus.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V2 consensus-blockstore-v2 $DATA_DIR/tendermint/data/blockstore.badger.db $OUTPUT_DIR/consensus-blockstore-v2.json > $OUTPUT_DIR/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v2.json > $OUTPUT_DIR/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 $DATA_DIR/tendermint/data/state.badger.db $OUTPUT_DIR/consensus-state-v2.json > $OUTPUT_DIR/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 $DATA_DIR/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT_DIR/consensus-mkvs-v2.json > $OUTPUT_DIR/consensus-mkvs-v2.log 2>&1 || true +post_run +) -# $OUTPUT/mainnet/20201118-20210428/ (BadgerDB v2) -mkdir -p "$OUTPUT/mainnet-20201118-20210428" +# mainnet-20201118-20210428 (BadgerDB v2) ( -$SAMPLER_V2 consensus-blockstore-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20201118-20210428/consensus-blockstore-v2.json > $OUTPUT/mainnet-20201118-20210428/consensus-blockstore-v2.log 2>&1 || true -$SAMPLER_V2 consensus-evidence-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20201118-20210428/consensus-evidence-v2.json > $OUTPUT/mainnet-20201118-20210428/consensus-evidence-v2.log 2>&1 || true -$SAMPLER_V2 consensus-state-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20201118-20210428/consensus-state-v2.json > $OUTPUT/mainnet-20201118-20210428/consensus-state-v2.log 2>&1 || true -$SAMPLER_V2 consensus-mkvs-v2 /snapshots/mainnet/20201118-20210428/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20201118-20210428/consensus-mkvs-v2.json > $OUTPUT/mainnet-20201118-20210428/consensus-mkvs-v2.log 2>&1 || true -) & +DATA_DIR="/snapshots/mainnet/20201118-20210428/consensus.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V2 consensus-blockstore-v2 $DATA_DIR/tendermint/data/blockstore.badger.db $OUTPUT_DIR/consensus-blockstore-v2.json > $OUTPUT_DIR/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v2.json > $OUTPUT_DIR/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 $DATA_DIR/tendermint/data/state.badger.db $OUTPUT_DIR/consensus-state-v2.json > $OUTPUT_DIR/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 $DATA_DIR/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT_DIR/consensus-mkvs-v2.json > $OUTPUT_DIR/consensus-mkvs-v2.log 2>&1 || true +post_run +) -# $OUTPUT/mainnet/20210428-20220411/ (BadgerDB v3) -mkdir -p "$OUTPUT/mainnet-20210428-20220411" +# mainnet-20210428-20220411 (BadgerDB v3) ( -$SAMPLER_V3 consensus-blockstore-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20210428-20220411/consensus-blockstore-v3.json > $OUTPUT/mainnet-20210428-20220411/consensus-blockstore-v3.log 2>&1 || true -$SAMPLER_V3 consensus-evidence-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20210428-20220411/consensus-evidence-v3.json > $OUTPUT/mainnet-20210428-20220411/consensus-evidence-v3.log 2>&1 || true -$SAMPLER_V3 consensus-state-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20210428-20220411/consensus-state-v3.json > $OUTPUT/mainnet-20210428-20220411/consensus-state-v3.log 2>&1 || true -$SAMPLER_V3 consensus-mkvs-v3 /snapshots/mainnet/20210428-20220411/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/consensus-mkvs-v3.json > $OUTPUT/mainnet-20210428-20220411/consensus-mkvs-v3.log 2>&1 || true -) & +DATA_DIR="/snapshots/mainnet/20210428-20220411/consensus.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V3 consensus-blockstore-v3 $DATA_DIR/tendermint/data/blockstore.badger.db $OUTPUT_DIR/consensus-blockstore-v3.json > $OUTPUT_DIR/consensus-blockstore-v3.log 2>&1 || true +$SAMPLER_V3 consensus-evidence-v3 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v3.json > $OUTPUT_DIR/consensus-evidence-v3.log 2>&1 || true +$SAMPLER_V3 consensus-state-v3 $DATA_DIR/tendermint/data/state.badger.db $OUTPUT_DIR/consensus-state-v3.json > $OUTPUT_DIR/consensus-state-v3.log 2>&1 || true +$SAMPLER_V3 consensus-mkvs-v3 $DATA_DIR/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT_DIR/consensus-mkvs-v3.json > $OUTPUT_DIR/consensus-mkvs-v3.log 2>&1 || true +post_run +) ( -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20210428-20220411/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/cipher-runtime-mkvs-v3.json > $OUTPUT/mainnet-20210428-20220411/cipher-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20210428-20220411/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT/mainnet-20210428-20220411/cipher-runtime-history-v3.json > $OUTPUT/mainnet-20210428-20220411/cipher-runtime-history-v3.log 2>&1 || true -) & +DATA_DIR="/snapshots/mainnet/20210428-20220411/cipher.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V3 runtime-mkvs-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT_DIR/cipher-runtime-mkvs-v3.json > $OUTPUT_DIR/cipher-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT_DIR/cipher-runtime-history-v3.json > $OUTPUT_DIR/cipher-runtime-history-v3.log 2>&1 || true +post_run +) ( -$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20210428-20220411/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT/mainnet-20210428-20220411/emerald-runtime-history-v3.json > $OUTPUT/mainnet-20210428-20220411/emerald-runtime-history-v3.log 2>&1 || true -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20210428-20220411/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT/mainnet-20210428-20220411/emerald-runtime-mkvs-v3.json > $OUTPUT/mainnet-20210428-20220411/emerald-runtime-mkvs-v3.log 2>&1 || true -) & +DATA_DIR="/snapshots/mainnet/20210428-20220411/emerald.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V3 runtime-history-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT_DIR/emerald-runtime-history-v3.json > $OUTPUT_DIR/emerald-runtime-history-v3.log 2>&1 || true +$SAMPLER_V3 runtime-mkvs-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT_DIR/emerald-runtime-mkvs-v3.json > $OUTPUT_DIR/emerald-runtime-mkvs-v3.log 2>&1 || true +post_run +) -# $OUTPUT/mainnet/20220411-20231129/ (BadgerDB v3) -mkdir -p "$OUTPUT/mainnet-20220411-20231129" +# mainnet-20220411-20231129 (BadgerDB v3) ( -$SAMPLER_V3 consensus-blockstore-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/blockstore.badger.db $OUTPUT/mainnet-20220411-20231129/consensus-blockstore-v3.json > $OUTPUT/mainnet-20220411-20231129/consensus-blockstore-v3.log 2>&1 || true -$SAMPLER_V3 consensus-evidence-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/evidence.badger.db $OUTPUT/mainnet-20220411-20231129/consensus-evidence-v3.json > $OUTPUT/mainnet-20220411-20231129/consensus-evidence-v3.log 2>&1 || true -$SAMPLER_V3 consensus-state-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/data/state.badger.db $OUTPUT/mainnet-20220411-20231129/consensus-state-v3.json > $OUTPUT/mainnet-20220411-20231129/consensus-state-v3.log 2>&1 || true -$SAMPLER_V3 consensus-mkvs-v3 /snapshots/mainnet/20220411-20231129/consensus.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/consensus-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/consensus-mkvs-v3.log 2>&1 || true -) & +DATA_DIR="/snapshots/mainnet/20220411-20231129/consensus.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V3 consensus-blockstore-v3 $DATA_DIR/tendermint/data/blockstore.badger.db $OUTPUT_DIR/consensus-blockstore-v3.json > $OUTPUT_DIR/consensus-blockstore-v3.log 2>&1 || true +$SAMPLER_V3 consensus-evidence-v3 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v3.json > $OUTPUT_DIR/consensus-evidence-v3.log 2>&1 || true +$SAMPLER_V3 consensus-state-v3 $DATA_DIR/tendermint/data/state.badger.db $OUTPUT_DIR/consensus-state-v3.json > $OUTPUT_DIR/consensus-state-v3.log 2>&1 || true +$SAMPLER_V3 consensus-mkvs-v3 $DATA_DIR/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT_DIR/consensus-mkvs-v3.json > $OUTPUT_DIR/consensus-mkvs-v3.log 2>&1 || true +post_run +) ( -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/cipher-runtime-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/cipher-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/cipher.mount/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT/mainnet-20220411-20231129/cipher-runtime-history-v3.json > $OUTPUT/mainnet-20220411-20231129/cipher-runtime-history-v3.log 2>&1 || true -) & +DATA_DIR="/snapshots/mainnet/20220411-20231129/cipher.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V3 runtime-mkvs-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT_DIR/cipher-runtime-mkvs-v3.json > $OUTPUT_DIR/cipher-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT_DIR/cipher-runtime-history-v3.json > $OUTPUT_DIR/cipher-runtime-history-v3.log 2>&1 || true +post_run +) ( -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/emerald-runtime-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/emerald-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/emerald.mount/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT/mainnet-20220411-20231129/emerald-runtime-history-v3.json > $OUTPUT/mainnet-20220411-20231129/emerald-runtime-history-v3.log 2>&1 || true -) & +DATA_DIR="/snapshots/mainnet/20220411-20231129/emerald.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V3 runtime-mkvs-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT_DIR/emerald-runtime-mkvs-v3.json > $OUTPUT_DIR/emerald-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT_DIR/emerald-runtime-history-v3.json > $OUTPUT_DIR/emerald-runtime-history-v3.log 2>&1 || true +post_run +) ( -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/mainnet/20220411-20231129/sapphire.mount/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/mkvs_storage.badger.db $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-mkvs-v3.json > $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/mainnet/20220411-20231129/sapphire.mount/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/history.db $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-history-v3.json > $OUTPUT/mainnet-20220411-20231129/sapphire-runtime-history-v3.log 2>&1 || true -) & +DATA_DIR="/snapshots/mainnet/20220411-20231129/sapphire.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V3 runtime-mkvs-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/mkvs_storage.badger.db $OUTPUT_DIR/sapphire-runtime-mkvs-v3.json > $OUTPUT_DIR/sapphire-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/history.db $OUTPUT_DIR/sapphire-runtime-history-v3.json > $OUTPUT_DIR/sapphire-runtime-history-v3.log 2>&1 || true +post_run +) -echo "All samplers launched in background!" -ps aux | grep badgerdb-sampler +echo "All samplers finished!" + +ls -ald outputs/*/* || true diff --git a/badgerdb-sampler/scripts/run-testnet-samplers.sh b/badgerdb-sampler/scripts/run-testnet-samplers.sh index d5df5f5..e66f60e 100755 --- a/badgerdb-sampler/scripts/run-testnet-samplers.sh +++ b/badgerdb-sampler/scripts/run-testnet-samplers.sh @@ -1,79 +1,117 @@ #!/bin/bash -# Script to run all badgerdb-samplers on Testnet +# Script to run all badgerdb-samplers on Testnet snapshots set -euo pipefail SAMPLER_V2="./bin/badgerdb-sampler-v2" SAMPLER_V3="./bin/badgerdb-sampler-v3" -OUTPUT="./outputs" +pre_run() { + mkdir -p "${OUTPUT_DIR}" + "$(dirname "${0}")"/snapshots-extract.sh --extract "${DATA_DIR%.dir}.tar.zst" + echo "Processing: ${DATA_DIR}..." +} +post_run() { + "$(dirname "${0}")"/snapshots-extract.sh --clean "${DATA_DIR%.dir}.tar.zst" + echo "Processing done." + echo +} -mkdir -p "$OUTPUT" -# $OUTPUT/testnet/20200915-20201104/ (BadgerDB v2) -mkdir -p "$OUTPUT/testnet-20200915-20201104" +# testnet-20200915-20201104 (BadgerDB v2) ( -$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20200915-20201104/consensus-blockstore-v2.json > $OUTPUT/testnet-20200915-20201104/consensus-blockstore-v2.log 2>&1 || true -$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20200915-20201104/consensus-evidence-v2.json > $OUTPUT/testnet-20200915-20201104/consensus-evidence-v2.log 2>&1 || true -$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20200915-20201104/consensus-state-v2.json > $OUTPUT/testnet-20200915-20201104/consensus-state-v2.log 2>&1 || true -$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20200915-20201104/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20200915-20201104/consensus-mkvs-v2.json > $OUTPUT/testnet-20200915-20201104/consensus-mkvs-v2.log 2>&1 || true -) & +DATA_DIR="/snapshots/testnet/20200915-20201104/consensus_testnet.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V2 consensus-blockstore-v2 $DATA_DIR/tendermint/data/blockstore.badger.db $OUTPUT_DIR/consensus-blockstore-v2.json > $OUTPUT_DIR/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v2.json > $OUTPUT_DIR/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 $DATA_DIR/tendermint/data/state.badger.db $OUTPUT_DIR/consensus-state-v2.json > $OUTPUT_DIR/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 $DATA_DIR/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT_DIR/consensus-mkvs-v2.json > $OUTPUT_DIR/consensus-mkvs-v2.log 2>&1 || true +post_run +) -# $OUTPUT/testnet/20201104-20210203/ (BadgerDB v2) -mkdir -p "$OUTPUT/testnet-20201104-20210203" +# testnet-20201104-20210203 (BadgerDB v2) ( -$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20201104-20210203/consensus-blockstore-v2.json > $OUTPUT/testnet-20201104-20210203/consensus-blockstore-v2.log 2>&1 || true -$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20201104-20210203/consensus-evidence-v2.json > $OUTPUT/testnet-20201104-20210203/consensus-evidence-v2.log 2>&1 || true -$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20201104-20210203/consensus-state-v2.json > $OUTPUT/testnet-20201104-20210203/consensus-state-v2.log 2>&1 || true -$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20201104-20210203/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20201104-20210203/consensus-mkvs-v2.json > $OUTPUT/testnet-20201104-20210203/consensus-mkvs-v2.log 2>&1 || true -) & +DATA_DIR="/snapshots/testnet/20201104-20210203/consensus_testnet.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V2 consensus-blockstore-v2 $DATA_DIR/tendermint/data/blockstore.badger.db $OUTPUT_DIR/consensus-blockstore-v2.json > $OUTPUT_DIR/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v2.json > $OUTPUT_DIR/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 $DATA_DIR/tendermint/data/state.badger.db $OUTPUT_DIR/consensus-state-v2.json > $OUTPUT_DIR/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 $DATA_DIR/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT_DIR/consensus-mkvs-v2.json > $OUTPUT_DIR/consensus-mkvs-v2.log 2>&1 || true +post_run +) -# $OUTPUT/testnet/20210203-20210324/ (BadgerDB v2) -mkdir -p "$OUTPUT/testnet-20210203-20210324" +# testnet-20210203-20210324 (BadgerDB v2) ( -$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210203-20210324/consensus-blockstore-v2.json > $OUTPUT/testnet-20210203-20210324/consensus-blockstore-v2.log 2>&1 || true -$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210203-20210324/consensus-evidence-v2.json > $OUTPUT/testnet-20210203-20210324/consensus-evidence-v2.log 2>&1 || true -$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210203-20210324/consensus-state-v2.json > $OUTPUT/testnet-20210203-20210324/consensus-state-v2.log 2>&1 || true -$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20210203-20210324/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20210203-20210324/consensus-mkvs-v2.json > $OUTPUT/testnet-20210203-20210324/consensus-mkvs-v2.log 2>&1 || true -) & +DATA_DIR="/snapshots/testnet/20210203-20210324/consensus_testnet.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V2 consensus-blockstore-v2 $DATA_DIR/tendermint/data/blockstore.badger.db $OUTPUT_DIR/consensus-blockstore-v2.json > $OUTPUT_DIR/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v2.json > $OUTPUT_DIR/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 $DATA_DIR/tendermint/data/state.badger.db $OUTPUT_DIR/consensus-state-v2.json > $OUTPUT_DIR/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 $DATA_DIR/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT_DIR/consensus-mkvs-v2.json > $OUTPUT_DIR/consensus-mkvs-v2.log 2>&1 || true +post_run +) -# $OUTPUT/testnet/20210324-20210413/ (BadgerDB v2) -mkdir -p "$OUTPUT/testnet-20210324-20210413" +# testnet-20210324-20210413 (BadgerDB v2) ( -$SAMPLER_V2 consensus-blockstore-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210324-20210413/consensus-blockstore-v2.json > $OUTPUT/testnet-20210324-20210413/consensus-blockstore-v2.log 2>&1 || true -$SAMPLER_V2 consensus-evidence-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210324-20210413/consensus-evidence-v2.json > $OUTPUT/testnet-20210324-20210413/consensus-evidence-v2.log 2>&1 || true -$SAMPLER_V2 consensus-state-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210324-20210413/consensus-state-v2.json > $OUTPUT/testnet-20210324-20210413/consensus-state-v2.log 2>&1 || true -$SAMPLER_V2 consensus-mkvs-v2 /snapshots/testnet/20210324-20210413/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20210324-20210413/consensus-mkvs-v2.json > $OUTPUT/testnet-20210324-20210413/consensus-mkvs-v2.log 2>&1 || true -) & +DATA_DIR="/snapshots/testnet/20210324-20210413/consensus_testnet.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V2 consensus-blockstore-v2 $DATA_DIR/tendermint/data/blockstore.badger.db $OUTPUT_DIR/consensus-blockstore-v2.json > $OUTPUT_DIR/consensus-blockstore-v2.log 2>&1 || true +$SAMPLER_V2 consensus-evidence-v2 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v2.json > $OUTPUT_DIR/consensus-evidence-v2.log 2>&1 || true +$SAMPLER_V2 consensus-state-v2 $DATA_DIR/tendermint/data/state.badger.db $OUTPUT_DIR/consensus-state-v2.json > $OUTPUT_DIR/consensus-state-v2.log 2>&1 || true +$SAMPLER_V2 consensus-mkvs-v2 $DATA_DIR/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT_DIR/consensus-mkvs-v2.json > $OUTPUT_DIR/consensus-mkvs-v2.log 2>&1 || true +post_run +) -# $OUTPUT/testnet/20210413-20220303/ (BadgerDB v3) -mkdir -p "$OUTPUT/testnet-20210413-20220303" +# testnet-20210413-20220303 (BadgerDB v3) ( -$SAMPLER_V3 consensus-blockstore-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20210413-20220303/consensus-blockstore-v3.json > $OUTPUT/testnet-20210413-20220303/consensus-blockstore-v3.log 2>&1 || true -$SAMPLER_V3 consensus-evidence-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20210413-20220303/consensus-evidence-v3.json > $OUTPUT/testnet-20210413-20220303/consensus-evidence-v3.log 2>&1 || true -$SAMPLER_V3 consensus-state-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20210413-20220303/consensus-state-v3.json > $OUTPUT/testnet-20210413-20220303/consensus-state-v3.log 2>&1 || true -$SAMPLER_V3 consensus-mkvs-v3 /snapshots/testnet/20210413-20220303/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20210413-20220303/consensus-mkvs-v3.json > $OUTPUT/testnet-20210413-20220303/consensus-mkvs-v3.log 2>&1 || true -) & +DATA_DIR="/snapshots/testnet/20210413-20220303/consensus_testnet.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V3 consensus-blockstore-v3 $DATA_DIR/tendermint/data/blockstore.badger.db _DIR-20220303/consensus-blockstore-v3.json > $OUTPUT_DIR/consensus-blockstore-v3.log 2>&1 || true +$SAMPLER_V3 consensus-evidence-v3 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v3.json > $OUTPUT_DIR/consensus-evidence-v3.log 2>&1 || true +$SAMPLER_V3 consensus-state-v3 $DATA_DIR/tendermint/data/state.badger.db $OUTPUT_DIR/consensus-state-v3.json > $OUTPUT_DIR/consensus-state-v3.log 2>&1 || true +$SAMPLER_V3 consensus-mkvs-v3 $DATA_DIR/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT_DIR/consensus-mkvs-v3.json > $OUTPUT_DIR/consensus-mkvs-v3.log 2>&1 || true +post_run +) -# $OUTPUT/testnet/20220303-20231012/ (BadgerDB v3) -mkdir -p "$OUTPUT/testnet-20220303-20231012" +# testnet-20220303-20231012 (BadgerDB v3) ( -$SAMPLER_V3 consensus-blockstore-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/blockstore.badger.db $OUTPUT/testnet-20220303-20231012/consensus-blockstore-v3.json > $OUTPUT/testnet-20220303-20231012/consensus-blockstore-v3.log 2>&1 || true -$SAMPLER_V3 consensus-evidence-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/evidence.badger.db $OUTPUT/testnet-20220303-20231012/consensus-evidence-v3.json > $OUTPUT/testnet-20220303-20231012/consensus-evidence-v3.log 2>&1 || true -$SAMPLER_V3 consensus-state-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/data/state.badger.db $OUTPUT/testnet-20220303-20231012/consensus-state-v3.json > $OUTPUT/testnet-20220303-20231012/consensus-state-v3.log 2>&1 || true -$SAMPLER_V3 consensus-mkvs-v3 /snapshots/testnet/20220303-20231012/consensus_testnet.mount/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/consensus-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/consensus-mkvs-v3.log 2>&1 || true -) & +DATA_DIR="/snapshots/testnet/20220303-20231012/consensus_testnet.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V3 consensus-blockstore-v3 $DATA_DIR/tendermint/data/blockstore.badger.db $OUTPUT_DIR/consensus-blockstore-v3.json > $OUTPUT_DIR/consensus-blockstore-v3.log 2>&1 || true +$SAMPLER_V3 consensus-evidence-v3 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v3.json > $OUTPUT_DIR/consensus-evidence-v3.log 2>&1 || true +$SAMPLER_V3 consensus-state-v3 $DATA_DIR/tendermint/data/state.badger.db $OUTPUT_DIR/consensus-state-v3.json > $OUTPUT_DIR/consensus-state-v3.log 2>&1 || true +$SAMPLER_V3 consensus-mkvs-v3 $DATA_DIR/tendermint/abci-state/mkvs_storage.badger.db $OUTPUT_DIR/consensus-mkvs-v3.json > $OUTPUT_DIR/consensus-mkvs-v3.log 2>&1 || true +post_run +) ( -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/cipher_testnet.mount/runtimes/0000000000000000000000000000000000000000000000000000000000000000/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/cipher-runtime-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/cipher-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/cipher_testnet.mount/runtimes/0000000000000000000000000000000000000000000000000000000000000000/history.db $OUTPUT/testnet-20220303-20231012/cipher-runtime-history-v3.json > $OUTPUT/testnet-20220303-20231012/cipher-runtime-history-v3.log 2>&1 || true -) & +DATA_DIR="/snapshots/testnet/20220303-20231012/cipher_testnet.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V3 runtime-mkvs-v3 $DATA_DIR/runtimes/0000000000000000000000000000000000000000000000000000000000000000/mkvs_storage.badger.db $OUTPUT_DIR/cipher-runtime-mkvs-v3.json > $OUTPUT_DIR/cipher-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 $DATA_DIR/runtimes/0000000000000000000000000000000000000000000000000000000000000000/history.db $OUTPUT_DIR/cipher-runtime-history-v3.json > $OUTPUT_DIR/cipher-runtime-history-v3.log 2>&1 || true +post_run +) ( -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/emerald_testnet.mount/runtimes/00000000000000000000000000000000000000000000000072c8215e60d5bca7/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/emerald-runtime-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/emerald-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/emerald_testnet.mount/runtimes/00000000000000000000000000000000000000000000000072c8215e60d5bca7/history.db $OUTPUT/testnet-20220303-20231012/emerald-runtime-history-v3.json > $OUTPUT/testnet-20220303-20231012/emerald-runtime-history-v3.log 2>&1 || true -) & +DATA_DIR="/snapshots/testnet/20220303-20231012/emerald_testnet.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V3 runtime-mkvs-v3 $DATA_DIR/runtimes/00000000000000000000000000000000000000000000000072c8215e60d5bca7/mkvs_storage.badger.db $OUTPUT_DIR/emerald-runtime-mkvs-v3.json > $OUTPUT_DIR/emerald-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 $DATA_DIR/runtimes/00000000000000000000000000000000000000000000000072c8215e60d5bca7/history.db $OUTPUT_DIR/emerald-runtime-history-v3.json > $OUTPUT_DIR/emerald-runtime-history-v3.log 2>&1 || true +post_run +) ( -$SAMPLER_V3 runtime-mkvs-v3 /snapshots/testnet/20220303-20231012/sapphire_testnet.mount/runtimes/000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c/mkvs_storage.badger.db $OUTPUT/testnet-20220303-20231012/sapphire-runtime-mkvs-v3.json > $OUTPUT/testnet-20220303-20231012/sapphire-runtime-mkvs-v3.log 2>&1 || true -$SAMPLER_V3 runtime-history-v3 /snapshots/testnet/20220303-20231012/sapphire_testnet.mount/runtimes/000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c/history.db $OUTPUT/testnet-20220303-20231012/sapphire-runtime-history-v3.json > $OUTPUT/testnet-20220303-20231012/sapphire-runtime-history-v3.log 2>&1 || true -) & +DATA_DIR="/snapshots/testnet/20220303-20231012/sapphire_testnet.dir" +OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +pre_run +$SAMPLER_V3 runtime-mkvs-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c/mkvs_storage.badger.db $OUTPUT_DIR/sapphire-runtime-mkvs-v3.json > $OUTPUT_DIR/sapphire-runtime-mkvs-v3.log 2>&1 || true +$SAMPLER_V3 runtime-history-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c/history.db $OUTPUT_DIR/sapphire-runtime-history-v3.json > $OUTPUT_DIR/sapphire-runtime-history-v3.log 2>&1 || true +post_run +) -echo "All samplers launched in background!" -echo "> ps aux | grep badgerdb-sampler" -ps aux | grep badgerdb-sampler +echo "All samplers finished!" + +ls -ald outputs/*/* || true diff --git a/badgerdb-sampler/scripts/snapshots-extract.sh b/badgerdb-sampler/scripts/snapshots-extract.sh new file mode 100755 index 0000000..79db016 --- /dev/null +++ b/badgerdb-sampler/scripts/snapshots-extract.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Script to extract or mount .tar.zst snapshot archives +# +# Usage: +# ./snapshots-extract.sh [--extract|--mount|--clean] [...] +# +# Examples: +# ./snapshots-extract.sh --extract testnet/*/*.tar.zst +# ./snapshots-extract.sh --mount testnet/*/*.tar.zst +# ./snapshots-extract.sh --clean testnet/*/*.tar.zst +set -euo pipefail + +usage() { + echo "Usage: $0 [--extract|--mount|--clean] [...]" + exit 1 +} + +mode="${1:-}" +if [[ "$mode" != "--extract" && "$mode" != "--mount" && "$mode" != "--clean" ]]; then + usage +fi +shift + +if [[ $# -eq 0 ]]; then + echo "Error: No snapshot files specified" + usage +fi + +for archive in "$@"; do + basename="${archive%.tar.zst}" + rootdir="${basename}.dir" + overlaydir="${basename}.overlay" + + if [[ "$mode" == "--extract" ]]; then + # Extract mode + if [[ -d "$rootdir" ]]; then + echo "Already extracted: $rootdir" + continue + fi + echo "Extracting: $archive -> $rootdir" + mkdir -p "$rootdir" + tar --zstd -C "$rootdir" -xf "$archive" + + elif [[ "$mode" == "--mount" ]]; then + # Mount mode + if mountpoint -q "$rootdir" 2>/dev/null; then + echo "Already mounted: $rootdir" + continue + fi + echo "Mounting: $archive -> $rootdir" + mkdir -p "$rootdir" + ./ratarmount -o allow_other,uid=1000,gid=1000 -w "$overlaydir" "$archive" "$rootdir" + + elif [[ "$mode" == "--clean" ]]; then + # Cleanup mode + echo "Cleaning: $basename" + if mountpoint -q "$rootdir" 2>/dev/null; then + timeout 120 fusermount -u "$rootdir" || sudo umount -f "$rootdir" + fi + rm -rf "$rootdir" "$overlaydir" + fi +done + +echo "Extracting done." diff --git a/badgerdb-sampler/scripts/umount-snapshots.sh b/badgerdb-sampler/scripts/umount-snapshots.sh deleted file mode 100755 index f31b0d0..0000000 --- a/badgerdb-sampler/scripts/umount-snapshots.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Script to unmount .tar.zst snapshot archives and remove overlays -set -euo pipefail - -cd "$(dirname "$0")" -search_dir="${1:-.}" - -find "$search_dir" -name "*.tar.zst" -type f | while read -r archive; do - basename="${archive%.tar.zst}" - mountpoint="${basename}.mount" - overlay="${basename}.overlay" - - if mountpoint -q "$mountpoint" 2>/dev/null; then - echo "Unmounting: $mountpoint" - fusermount -u "$mountpoint" - rm -rf "$mountpoint" "$overlay" - fi -done - -echo "Done!" From 23165bf6462b50368ef6964adc1209d37dd3ff1e Mon Sep 17 00:00:00 2001 From: gw0 Date: Mon, 22 Dec 2025 13:51:01 +0100 Subject: [PATCH 22/22] badgerdb-sampler: Various fixes --- badgerdb-sampler/README.md | 4 +++- .../scripts/run-mainnet-samplers.sh | 18 +++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/badgerdb-sampler/README.md b/badgerdb-sampler/README.md index 17cf9f3..bace558 100644 --- a/badgerdb-sampler/README.md +++ b/badgerdb-sampler/README.md @@ -2,6 +2,8 @@ This **badgerdb-sampler** tool implements a comprehensive data extraction and decoding logic for BadgerDB databases used in Oasis node snapshots. This tool can extract and decode most data structures from consensus and runtime databases, even EVM data. It handles multiple BadgerDB versions (v2-v4) and data representations use by Oasis nodes (v20.x-v25.x). The decoding logic handles various data formats and reports decoding issues. +**Warning:** For experimental purposes only. Decoded data might be incorrect. + ## Features - **Multi-version BadgerDB Support**: Compatible with BadgerDB v2, v3, and v4 @@ -19,8 +21,8 @@ This **badgerdb-sampler** tool implements a comprehensive data extraction and de - `consensus-evidence` - Byzantine validator evidence - `consensus-mkvs` - Consensus state Merkle tree - `consensus-state` - Tendermint/CometBFT consensus state -- `runtime-mkvs` - Runtime state Merkle tree (includes EVM storage) - `runtime-history` - Runtime block history with CBOR-encoded data (includes EVM events/transactions) +- `runtime-mkvs` - Runtime state Merkle tree (includes EVM storage) ## Building diff --git a/badgerdb-sampler/scripts/run-mainnet-samplers.sh b/badgerdb-sampler/scripts/run-mainnet-samplers.sh index 3998b29..28fd372 100755 --- a/badgerdb-sampler/scripts/run-mainnet-samplers.sh +++ b/badgerdb-sampler/scripts/run-mainnet-samplers.sh @@ -19,7 +19,7 @@ post_run() { # mainnet-20201001-20201118 (BadgerDB v2) ( DATA_DIR="/snapshots/mainnet/20201001-20201118/consensus.dir" -OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +OUTPUT_DIR="./outputs/mainnet-$(basename $(dirname "${DATA_DIR}"))" pre_run $SAMPLER_V2 consensus-blockstore-v2 $DATA_DIR/tendermint/data/blockstore.badger.db $OUTPUT_DIR/consensus-blockstore-v2.json > $OUTPUT_DIR/consensus-blockstore-v2.log 2>&1 || true $SAMPLER_V2 consensus-evidence-v2 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v2.json > $OUTPUT_DIR/consensus-evidence-v2.log 2>&1 || true @@ -31,7 +31,7 @@ post_run # mainnet-20201118-20210428 (BadgerDB v2) ( DATA_DIR="/snapshots/mainnet/20201118-20210428/consensus.dir" -OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +OUTPUT_DIR="./outputs/mainnet-$(basename $(dirname "${DATA_DIR}"))" pre_run $SAMPLER_V2 consensus-blockstore-v2 $DATA_DIR/tendermint/data/blockstore.badger.db $OUTPUT_DIR/consensus-blockstore-v2.json > $OUTPUT_DIR/consensus-blockstore-v2.log 2>&1 || true $SAMPLER_V2 consensus-evidence-v2 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v2.json > $OUTPUT_DIR/consensus-evidence-v2.log 2>&1 || true @@ -43,7 +43,7 @@ post_run # mainnet-20210428-20220411 (BadgerDB v3) ( DATA_DIR="/snapshots/mainnet/20210428-20220411/consensus.dir" -OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +OUTPUT_DIR="./outputs/mainnet-$(basename $(dirname "${DATA_DIR}"))" pre_run $SAMPLER_V3 consensus-blockstore-v3 $DATA_DIR/tendermint/data/blockstore.badger.db $OUTPUT_DIR/consensus-blockstore-v3.json > $OUTPUT_DIR/consensus-blockstore-v3.log 2>&1 || true $SAMPLER_V3 consensus-evidence-v3 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v3.json > $OUTPUT_DIR/consensus-evidence-v3.log 2>&1 || true @@ -53,7 +53,7 @@ post_run ) ( DATA_DIR="/snapshots/mainnet/20210428-20220411/cipher.dir" -OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +OUTPUT_DIR="./outputs/mainnet-$(basename $(dirname "${DATA_DIR}"))" pre_run $SAMPLER_V3 runtime-mkvs-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT_DIR/cipher-runtime-mkvs-v3.json > $OUTPUT_DIR/cipher-runtime-mkvs-v3.log 2>&1 || true $SAMPLER_V3 runtime-history-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT_DIR/cipher-runtime-history-v3.json > $OUTPUT_DIR/cipher-runtime-history-v3.log 2>&1 || true @@ -61,7 +61,7 @@ post_run ) ( DATA_DIR="/snapshots/mainnet/20210428-20220411/emerald.dir" -OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +OUTPUT_DIR="./outputs/mainnet-$(basename $(dirname "${DATA_DIR}"))" pre_run $SAMPLER_V3 runtime-history-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT_DIR/emerald-runtime-history-v3.json > $OUTPUT_DIR/emerald-runtime-history-v3.log 2>&1 || true $SAMPLER_V3 runtime-mkvs-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT_DIR/emerald-runtime-mkvs-v3.json > $OUTPUT_DIR/emerald-runtime-mkvs-v3.log 2>&1 || true @@ -71,7 +71,7 @@ post_run # mainnet-20220411-20231129 (BadgerDB v3) ( DATA_DIR="/snapshots/mainnet/20220411-20231129/consensus.dir" -OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +OUTPUT_DIR="./outputs/mainnet-$(basename $(dirname "${DATA_DIR}"))" pre_run $SAMPLER_V3 consensus-blockstore-v3 $DATA_DIR/tendermint/data/blockstore.badger.db $OUTPUT_DIR/consensus-blockstore-v3.json > $OUTPUT_DIR/consensus-blockstore-v3.log 2>&1 || true $SAMPLER_V3 consensus-evidence-v3 $DATA_DIR/tendermint/data/evidence.badger.db $OUTPUT_DIR/consensus-evidence-v3.json > $OUTPUT_DIR/consensus-evidence-v3.log 2>&1 || true @@ -81,7 +81,7 @@ post_run ) ( DATA_DIR="/snapshots/mainnet/20220411-20231129/cipher.dir" -OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +OUTPUT_DIR="./outputs/mainnet-$(basename $(dirname "${DATA_DIR}"))" pre_run $SAMPLER_V3 runtime-mkvs-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/mkvs_storage.badger.db $OUTPUT_DIR/cipher-runtime-mkvs-v3.json > $OUTPUT_DIR/cipher-runtime-mkvs-v3.log 2>&1 || true $SAMPLER_V3 runtime-history-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e199119c992377cb/history.db $OUTPUT_DIR/cipher-runtime-history-v3.json > $OUTPUT_DIR/cipher-runtime-history-v3.log 2>&1 || true @@ -89,7 +89,7 @@ post_run ) ( DATA_DIR="/snapshots/mainnet/20220411-20231129/emerald.dir" -OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +OUTPUT_DIR="./outputs/mainnet-$(basename $(dirname "${DATA_DIR}"))" pre_run $SAMPLER_V3 runtime-mkvs-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/mkvs_storage.badger.db $OUTPUT_DIR/emerald-runtime-mkvs-v3.json > $OUTPUT_DIR/emerald-runtime-mkvs-v3.log 2>&1 || true $SAMPLER_V3 runtime-history-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000e2eaa99fc008f87f/history.db $OUTPUT_DIR/emerald-runtime-history-v3.json > $OUTPUT_DIR/emerald-runtime-history-v3.log 2>&1 || true @@ -97,7 +97,7 @@ post_run ) ( DATA_DIR="/snapshots/mainnet/20220411-20231129/sapphire.dir" -OUTPUT_DIR="./outputs/testnet-$(basename $(dirname "${DATA_DIR}"))" +OUTPUT_DIR="./outputs/mainnet-$(basename $(dirname "${DATA_DIR}"))" pre_run $SAMPLER_V3 runtime-mkvs-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/mkvs_storage.badger.db $OUTPUT_DIR/sapphire-runtime-mkvs-v3.json > $OUTPUT_DIR/sapphire-runtime-mkvs-v3.log 2>&1 || true $SAMPLER_V3 runtime-history-v3 $DATA_DIR/runtimes/000000000000000000000000000000000000000000000000f80306c9858e7279/history.db $OUTPUT_DIR/sapphire-runtime-history-v3.json > $OUTPUT_DIR/sapphire-runtime-history-v3.log 2>&1 || true