From ffd62dce19ed4f4211fa9e66c0600acbea59d569 Mon Sep 17 00:00:00 2001 From: Artem Klevtsov Date: Sun, 5 Oct 2025 12:13:51 +0700 Subject: [PATCH 01/51] Add template_processor --- internal/impl/pure/processor_template.go | 165 +++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 internal/impl/pure/processor_template.go diff --git a/internal/impl/pure/processor_template.go b/internal/impl/pure/processor_template.go new file mode 100644 index 000000000..6f5ad1b00 --- /dev/null +++ b/internal/impl/pure/processor_template.go @@ -0,0 +1,165 @@ +// Copyright 2025 Redpanda Data, Inc. + +package pure + +import ( + "bytes" + "context" + "errors" + "text/template" + + "github.com/redpanda-data/benthos/v4/internal/bundle" + "github.com/redpanda-data/benthos/v4/internal/component/interop" + "github.com/redpanda-data/benthos/v4/public/bloblang" + "github.com/redpanda-data/benthos/v4/public/service" +) + +func tmplProcConfig() *service.ConfigSpec { + return service.NewConfigSpec(). + Beta(). + Categories("Utility"). + Summary("Executes a Go text/template template on the message content."). + Description(`This processor allows you to apply Go text/template templates to the structured content of messages. The template can access the message data as a structured object. Optionally, a Bloblang mapping can be applied first to transform the data before templating. + +For more information on the template syntax, see https://pkg.go.dev/text/template#hdr-Actions`). + Example( + "Execute template", + `This example uses a xref:components:inputs/generate.adoc[`+"`generate`"+` input] to make payload for the template.`, + ` +input: + generate: + count: 1 + mapping: root.foo = "bar" + processors: + - template: + code: "{{ .foo }}" +`). + Example( + "Execute template with mapping", + `This example uses a xref:components:inputs/generate.adoc[`+"`generate`"+` input] to make payload for the template.`, + ` +input: + generate: + count: 1 + mapping: root.foo = "bar" + processors: + - template: + code: "{{ .value }}" + mapping: "root.value = this.foo" +`). + Example( + "Execute template from file", + `This example loads a template from a file and applies it to the message.`, + ` +input: + generate: + count: 1 + mapping: root.foo = "bar" + processors: + - template: + code: | + {{ template "greeting" . }} + files: ["./templates/greeting.tmpl"] +`). + Fields( + service.NewStringField("code"). + Description("The template code to execute. This should be a valid Go text/template string."). + Example("{{.name}}"). + Optional(), + service.NewStringListField("files"). + Description("A list of file paths containing template definitions. Templates from these files will be parsed and available for execution."). + Optional(), + service.NewBloblangField("mapping"). + Description("An optional xref:guides:bloblang/about.adoc[Bloblang] mapping to apply to the message before executing the template. This allows you to transform the data structure before templating."). + Optional(), + ) +} + +func init() { + service.MustRegisterProcessor( + "template", + tmplProcConfig(), + func(conf *service.ParsedConfig, res *service.Resources) (service.Processor, error) { + mgr := interop.UnwrapManagement(res) + return templateFromParsed(conf, mgr) + }, + ) +} + +type tmplProc struct { + tmpl *template.Template + exec *bloblang.Executor +} + +func templateFromParsed(conf *service.ParsedConfig, mgr bundle.NewManagement) (*tmplProc, error) { + code, err := conf.FieldString("code") + if err != nil { + return nil, err + } + + files, err := conf.FieldStringList("files") + if err != nil { + return nil, err + } + + if code == "" && len(files) == 0 { + return nil, errors.New("code or files param must be specified") + } + + t := &tmplProc{tmpl: &template.Template{}} + if len(files) > 0 { + if t.tmpl, err = t.tmpl.ParseFiles(files...); err != nil { + return nil, err + } + } + + if code != "" { + if t.tmpl, err = t.tmpl.New("code").Parse(code); err != nil { + return nil, err + } + } + + if conf.Contains("mapping") { + if t.exec, err = conf.FieldBloblang("mapping"); err != nil { + return nil, err + } + } + + return t, nil +} + +func (t *tmplProc) Process(ctx context.Context, msg *service.Message) (service.MessageBatch, error) { + var data any + var err error + if t.exec != nil { + mapRes, err := msg.BloblangQuery(t.exec) + if err != nil { + return nil, err + } + + data, err = mapRes.AsStructured() + if err != nil { + return nil, err + } + } else { + data, err = msg.AsStructured() + if err != nil { + return nil, err + } + } + + var buf bytes.Buffer + if err := t.tmpl.Execute(&buf, data); err != nil { + return nil, err + } + + msg.SetBytes(buf.Bytes()) + + return service.MessageBatch{msg}, nil +} + +func (t *tmplProc) Close(ctx context.Context) error { + _, err := t.tmpl.Clone() + + return err +} From e0c76f88a6f65b471a53c8bf11bb65e89e109c05 Mon Sep 17 00:00:00 2001 From: Artem Klevtsov Date: Sun, 5 Oct 2025 12:32:54 +0700 Subject: [PATCH 02/51] Add glob support to the template processor --- internal/impl/pure/processor_template.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/internal/impl/pure/processor_template.go b/internal/impl/pure/processor_template.go index 6f5ad1b00..f9f600220 100644 --- a/internal/impl/pure/processor_template.go +++ b/internal/impl/pure/processor_template.go @@ -39,10 +39,10 @@ input: `This example uses a xref:components:inputs/generate.adoc[`+"`generate`"+` input] to make payload for the template.`, ` input: - generate: + generate: count: 1 mapping: root.foo = "bar" - processors: + processors: - template: code: "{{ .value }}" mapping: "root.value = this.foo" @@ -52,13 +52,13 @@ input: `This example loads a template from a file and applies it to the message.`, ` input: - generate: + generate: count: 1 mapping: root.foo = "bar" - processors: - - template: - code: | - {{ template "greeting" . }} + processors: + - template: + code: | + {{ template "greeting" . }} files: ["./templates/greeting.tmpl"] `). Fields( @@ -67,7 +67,7 @@ input: Example("{{.name}}"). Optional(), service.NewStringListField("files"). - Description("A list of file paths containing template definitions. Templates from these files will be parsed and available for execution."). + Description("A list of file paths containing template definitions. Templates from these files will be parsed and available for execution. Glob patterns are supported, including super globs (double star)."). Optional(), service.NewBloblangField("mapping"). Description("An optional xref:guides:bloblang/about.adoc[Bloblang] mapping to apply to the message before executing the template. This allows you to transform the data structure before templating."). @@ -108,8 +108,10 @@ func templateFromParsed(conf *service.ParsedConfig, mgr bundle.NewManagement) (* t := &tmplProc{tmpl: &template.Template{}} if len(files) > 0 { - if t.tmpl, err = t.tmpl.ParseFiles(files...); err != nil { - return nil, err + for _, f := range files { + if t.tmpl, err = t.tmpl.ParseGlob(f); err != nil { + return nil, err + } } } From 0130ea9e754c7c4fda48e580a8c09fd4960109de Mon Sep 17 00:00:00 2001 From: Artem Klevtsov Date: Tue, 9 Dec 2025 00:23:01 +0700 Subject: [PATCH 03/51] fix: tmplProc.Close method --- internal/impl/pure/processor_template.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/impl/pure/processor_template.go b/internal/impl/pure/processor_template.go index f9f600220..00f513ab7 100644 --- a/internal/impl/pure/processor_template.go +++ b/internal/impl/pure/processor_template.go @@ -161,7 +161,5 @@ func (t *tmplProc) Process(ctx context.Context, msg *service.Message) (service.M } func (t *tmplProc) Close(ctx context.Context) error { - _, err := t.tmpl.Clone() - - return err + return nil } From 7240afee5dbb370ee16642cf7d362e212c004bb8 Mon Sep 17 00:00:00 2001 From: Artem Klevtsov Date: Tue, 9 Dec 2025 00:23:38 +0700 Subject: [PATCH 04/51] fix: template init --- internal/impl/pure/processor_template.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/pure/processor_template.go b/internal/impl/pure/processor_template.go index 00f513ab7..dd4f74488 100644 --- a/internal/impl/pure/processor_template.go +++ b/internal/impl/pure/processor_template.go @@ -106,7 +106,7 @@ func templateFromParsed(conf *service.ParsedConfig, mgr bundle.NewManagement) (* return nil, errors.New("code or files param must be specified") } - t := &tmplProc{tmpl: &template.Template{}} + t := &tmplProc{tmpl: template.New("root")} if len(files) > 0 { for _, f := range files { if t.tmpl, err = t.tmpl.ParseGlob(f); err != nil { From 37e751447257576a28fdf30ea90a24a368a609c7 Mon Sep 17 00:00:00 2001 From: Artem Klevtsov Date: Tue, 9 Dec 2025 00:25:52 +0700 Subject: [PATCH 05/51] fix templateFromParsed error message --- internal/impl/pure/processor_template.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/pure/processor_template.go b/internal/impl/pure/processor_template.go index dd4f74488..1ef92b3ae 100644 --- a/internal/impl/pure/processor_template.go +++ b/internal/impl/pure/processor_template.go @@ -103,7 +103,7 @@ func templateFromParsed(conf *service.ParsedConfig, mgr bundle.NewManagement) (* } if code == "" && len(files) == 0 { - return nil, errors.New("code or files param must be specified") + return nil, errors.New("at least one of 'code' or 'files' fields must be specified") } t := &tmplProc{tmpl: template.New("root")} From f6f7d9b6c56f12b70aa1e1c21379dd2989297d0e Mon Sep 17 00:00:00 2001 From: Artem Klevtsov Date: Tue, 9 Dec 2025 00:29:23 +0700 Subject: [PATCH 06/51] fix: config examples --- internal/impl/pure/processor_template.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/impl/pure/processor_template.go b/internal/impl/pure/processor_template.go index 1ef92b3ae..89965292f 100644 --- a/internal/impl/pure/processor_template.go +++ b/internal/impl/pure/processor_template.go @@ -30,6 +30,8 @@ input: generate: count: 1 mapping: root.foo = "bar" + +pipeline: processors: - template: code: "{{ .foo }}" @@ -42,6 +44,8 @@ input: generate: count: 1 mapping: root.foo = "bar" + +pipeline: processors: - template: code: "{{ .value }}" From bf033ed5bc10923eee6f0604700cfab567873551 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Fri, 10 Oct 2025 11:12:42 +0100 Subject: [PATCH 07/51] chore: bump x/crypto package to 0.43.0 (#293) --- go.mod | 8 ++++---- go.sum | 28 ++++++++++++++-------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index a8ef571d4..728d2ee6c 100644 --- a/go.mod +++ b/go.mod @@ -40,10 +40,10 @@ require ( go.opentelemetry.io/otel v1.37.0 go.opentelemetry.io/otel/trace v1.37.0 go.uber.org/multierr v1.11.0 - golang.org/x/crypto v0.39.0 + golang.org/x/crypto v0.43.0 golang.org/x/oauth2 v0.30.0 - golang.org/x/sync v0.15.0 - golang.org/x/text v0.26.0 + golang.org/x/sync v0.17.0 + golang.org/x/text v0.30.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -69,7 +69,7 @@ require ( github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel/metric v1.37.0 // indirect - golang.org/x/sys v0.33.0 // indirect + golang.org/x/sys v0.37.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 5b8ce0dcb..339bbdb00 100644 --- a/go.sum +++ b/go.sum @@ -166,29 +166,29 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= -golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= -golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= +golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM= +golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= -golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= -golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= +golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 6c3ba5a78f6b97a3d6d6c5ca609ba037af63ef6e Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Mon, 13 Oct 2025 09:18:21 +0100 Subject: [PATCH 08/51] chore: bump Go from 1.25.1 to 1.25.2 (#295) --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 728d2ee6c..c54ee6959 100644 --- a/go.mod +++ b/go.mod @@ -73,4 +73,4 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect ) -go 1.25.1 +go 1.25.2 From 76dfb204cf78bddf5a3f340ebe4b449f0e3ad76e Mon Sep 17 00:00:00 2001 From: Alex Treichler Date: Thu, 16 Oct 2025 01:47:21 -0700 Subject: [PATCH 09/51] netclient: introduce package to for custom TCP settings --- public/netclient/client_linux.go | 38 +++++++++++++ public/netclient/client_other.go | 22 ++++++++ public/netclient/config.go | 94 ++++++++++++++++++++++++++++++++ public/netclient/config_test.go | 89 ++++++++++++++++++++++++++++++ public/netclient/spec.go | 73 +++++++++++++++++++++++++ 5 files changed, 316 insertions(+) create mode 100644 public/netclient/client_linux.go create mode 100644 public/netclient/client_other.go create mode 100644 public/netclient/config.go create mode 100644 public/netclient/config_test.go create mode 100644 public/netclient/spec.go diff --git a/public/netclient/client_linux.go b/public/netclient/client_linux.go new file mode 100644 index 000000000..3dff2e1fb --- /dev/null +++ b/public/netclient/client_linux.go @@ -0,0 +1,38 @@ +//go:build linux + +// Copyright 2025 Redpanda Data, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package netclient + +import ( + "fmt" + "syscall" +) + +// tcpUserTimeout is a linux specific constant that is used to reference +// the tcp_user_timeout option. +const tcpUserTimeout = 18 + +// SetTCPUserTimeout sets the "TCP_USER_TIMEOUT" socket option on Linux. +func (c *Config) setTCPUserTimeout(fd int) error { + timeoutMs := int(c.TCPUserTimeout.Milliseconds()) + + err := syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, tcpUserTimeout, timeoutMs) + if err != nil { + return fmt.Errorf("failed to set tcp_user_timeout: %w", err) + } + + return nil +} diff --git a/public/netclient/client_other.go b/public/netclient/client_other.go new file mode 100644 index 000000000..9cd42f8de --- /dev/null +++ b/public/netclient/client_other.go @@ -0,0 +1,22 @@ +//go:build !linux + +// Copyright 2025 Redpanda Data, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package netclient + +// SetTCPUserTimeout does not apply to non-linux systems as it is not +// supported on other platforms. Errors and warnings will be silently ignored. +func (c *Config) setTCPUserTimeout(_ int) error { + return nil +} diff --git a/public/netclient/config.go b/public/netclient/config.go new file mode 100644 index 000000000..c943179e9 --- /dev/null +++ b/public/netclient/config.go @@ -0,0 +1,94 @@ +// Copyright 2025 Redpanda Data, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package netclient + +import ( + "fmt" + "net" + "syscall" + "time" +) + +// Config contains TCP socket configuration options. +// TCPUserTimeout is only supported on Linux +// since 2.6.37 (https://www.man7.org/linux/man-pages/man7/tcp.7.html). +// On other platforms it is ignored. +type Config struct { + KeepAliveConfig net.KeepAliveConfig + TCPUserTimeout time.Duration +} + +// Validate checks that the configuration is valid. +func (config Config) Validate() error { + // KeepAlive MUST be greater than TCP_USER_TIMEOUT + // per RFC 5482 (https://www.rfc-editor.org/rfc/rfc5482.html). + if config.TCPUserTimeout > 0 && config.KeepAliveConfig.Idle <= config.TCPUserTimeout { + return fmt.Errorf("keep_alive.idle (%s) must be greater than tcp_user_timeout (%s)", config.KeepAliveConfig.Idle, config.TCPUserTimeout) + } + return nil +} + +// NewDialerFrom creates a new net.Dialer from the provided Config. +// It validates the Config and returns an error if validation fails. +// The returned Dialer will have TCP options applied according to the Config. +func NewDialerFrom(config Config) (*net.Dialer, error) { + if err := config.Validate(); err != nil { + return nil, err + } + return config.newDialer(), nil +} + +// newDialer returns a net.Dialer configured with the TCP options from the +// Config. +func (config Config) newDialer() *net.Dialer { + dialer := &net.Dialer{ + KeepAliveConfig: config.KeepAliveConfig, + } + + if controlFn := config.controlFunc(); controlFn != nil { + dialer.Control = controlFn + } + return dialer +} + +// controlFunc returns a function that configures TCP socket options. +func (config Config) controlFunc() func(network, address string, con syscall.RawConn) error { + // Do not do anything if tcp_user_timeout is not set. + if config.TCPUserTimeout <= 0 { + return nil + } + return func(network, address string, conn syscall.RawConn) error { + var setErr error + // Starting connection to the specific file descriptor. + err := conn.Control(func(fd uintptr) { + // Set timeout. + if err := config.setTCPUserTimeout(int(fd)); err != nil { + setErr = err + return + } + }) + if err != nil { + // If no connection was able to be established then return error and + // what network and address it is trying to connect to. + return fmt.Errorf("failed to access raw connection for: %s %s: %w", network, address, err) + } + if setErr != nil { + // If connection was established, but we were unable to set the timeout + // for some reason. + return fmt.Errorf("failed to set TCP_USER_TIMEOUT (%v) on %s %s: %w", config.TCPUserTimeout, network, address, setErr) + } + return nil + } +} diff --git a/public/netclient/config_test.go b/public/netclient/config_test.go new file mode 100644 index 000000000..881d17fca --- /dev/null +++ b/public/netclient/config_test.go @@ -0,0 +1,89 @@ +// Copyright 2025 Redpanda Data, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package netclient + +import ( + "net" + "testing" + "time" +) + +func TestValidate(t *testing.T) { + tests := []struct { + name string + config Config + wantErr bool + }{ + { + name: "No TCPUserTimeout", + config: Config{ + TCPUserTimeout: 0, + KeepAliveConfig: net.KeepAliveConfig{ + Idle: 10 * time.Second, + }, + }, + wantErr: false, + }, + { + // Default value is 15seconds. + name: "TCPUserTimeout set, but KeepAlive idle not set", + config: Config{ + TCPUserTimeout: 10 * time.Second, + KeepAliveConfig: net.KeepAliveConfig{ + Idle: 15 * time.Second, + }, + }, + wantErr: false, + }, + { + name: "KeepAlive idle less than TCPUserTimeout", + config: Config{ + TCPUserTimeout: 10 * time.Second, + KeepAliveConfig: net.KeepAliveConfig{ + Idle: 5 * time.Second, + }, + }, + wantErr: true, + }, + { + name: "KeepAlive idle equal to TCPUserTimeout", + config: Config{ + TCPUserTimeout: 10 * time.Second, + KeepAliveConfig: net.KeepAliveConfig{ + Idle: 10 * time.Second, + }, + }, + wantErr: true, + }, + { + name: "KeepAlive idle greater than TCPUserTimeout", + config: Config{ + TCPUserTimeout: 10 * time.Second, + KeepAliveConfig: net.KeepAliveConfig{ + Idle: 30 * time.Second, + }, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.config.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Config.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/public/netclient/spec.go b/public/netclient/spec.go new file mode 100644 index 000000000..f0a1b3e2d --- /dev/null +++ b/public/netclient/spec.go @@ -0,0 +1,73 @@ +// Copyright 2025 Redpanda Data, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package netclient + +import "github.com/redpanda-data/benthos/v4/public/service" + +// ConfigSpec returns the config spec for TCP options. +func ConfigSpec() *service.ConfigField { + return service.NewObjectField("tcp", + service.NewObjectField("keep_alive", + service.NewDurationField("idle"). + Description("Idle is the time that the connection must be idle before the first keep-alive probe is sent. If zero, a default value of 15 seconds is used. If negative, then keep-alive probes are disabled."). + Default("15s"), + service.NewDurationField("interval"). + Description("Interval is the time between keep-alive probes. If zero, a default value of 15 seconds is used."). + Default("15s"), + service.NewIntField("count"). + Description("Count is the maximum number of keep-alive probes that can go unanswered before dropping a connection. If zero, a default value of 9 is used"). + Default(9), + ).Description("Custom TCP keep-alive probe configuration."). + Optional(), + service.NewDurationField("tcp_user_timeout"). + Description("Linux-specific TCP_USER_TIMEOUT defines how long to wait for acknowledgment of transmitted data on an established connection before killing the connection. This allows more fine grained control on the application level as opposed to the system-wide kernel setting, tcp_retries2. If enabled, keep_alive.idle must be set to a greater value. Set to 0 (default) to disable."). + Default("0s"), + ). + Description("TCP socket configuration options"). + Optional() +} + +// ParseConfig parses a namespaced TCP config. +func ParseConfig(pConf *service.ParsedConfig) (Config, error) { + cfg := Config{} + var err error + + cfg.TCPUserTimeout, err = pConf.FieldDuration("tcp_user_timeout") + if err != nil { + return cfg, err + } + + if pConf.Contains("keep_alive") { + pc := pConf.Namespace("keep_alive") + // Each field is optional, so ignoring errors. + if idle, err := pc.FieldDuration("keep_alive"); err == nil { + cfg.KeepAliveConfig.Idle = idle + } + if interval, err := pc.FieldDuration("interval"); err == nil { + cfg.KeepAliveConfig.Interval = interval + } + if count, err := pc.FieldInt("count"); err == nil { + cfg.KeepAliveConfig.Count = count + } + // If KeepAliveConfig.Idle is a negative number then we assume they want + // KeepAlives disabled as outlined in the idle description. + if cfg.KeepAliveConfig.Idle >= 0 { + cfg.KeepAliveConfig.Enable = true + } else { + cfg.KeepAliveConfig.Enable = false + } + } + return cfg, nil +} From 37e3a2945063974c47e1eaf1c5b06cba8b97caa0 Mon Sep 17 00:00:00 2001 From: Michal Matczuk Date: Tue, 21 Oct 2025 14:23:08 +0200 Subject: [PATCH 10/51] message: preallocate metadata (#300) --- internal/message/data.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/message/data.go b/internal/message/data.go index d03e3e1d2..f81f5cd13 100644 --- a/internal/message/data.go +++ b/internal/message/data.go @@ -16,6 +16,10 @@ type messageData struct { metadata map[string]any } +// defaultMetadataSize specifies how many metadata entries are preallocated by +// default when adding metadata to a message. +const defaultMetadataSize = 5 + func newMessageBytes(content []byte) *messageData { return &messageData{ rawBytes: content, @@ -180,10 +184,7 @@ func (m *messageData) MetaGetMut(key string) (any, bool) { func (m *messageData) MetaSetMut(key string, value any) { m.writeableMeta() if m.metadata == nil { - m.metadata = map[string]any{ - key: value, - } - return + m.metadata = make(map[string]any, defaultMetadataSize) } m.metadata[key] = value } From af6ccff9478146aea114e15d2483249706541dcd Mon Sep 17 00:00:00 2001 From: tomasz-sadura Date: Wed, 22 Oct 2025 14:47:05 +0200 Subject: [PATCH 11/51] Extend field JSON schema (#301) --- internal/docs/json_schema.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/docs/json_schema.go b/internal/docs/json_schema.go index 3726849a1..82e73f4d7 100644 --- a/internal/docs/json_schema.go +++ b/internal/docs/json_schema.go @@ -20,6 +20,11 @@ func jSchemaIsRequired(f *FieldSpec) bool { // JSONSchema serializes a field spec into a JSON schema structure. func (f FieldSpec) JSONSchema() any { spec := map[string]any{} + spec["is_advanced"] = f.IsAdvanced + spec["is_deprecated"] = f.IsDeprecated + spec["is_optional"] = f.IsOptional + spec["is_secret"] = f.IsSecret + spec["version"] = f.Version switch f.Kind { case Kind2DArray: innerField := f From d625fe9cd5438c39534a8ffbec02fcb53de98923 Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Thu, 23 Oct 2025 10:28:14 +0100 Subject: [PATCH 12/51] Allow deprecated templates (#302) * Allow templates to be marked deprecated * Remove version field from jsonspec --- internal/docs/json_schema.go | 1 - internal/template/config.go | 1 + internal/template/template_test.go | 41 ++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/internal/docs/json_schema.go b/internal/docs/json_schema.go index 82e73f4d7..727c88b9a 100644 --- a/internal/docs/json_schema.go +++ b/internal/docs/json_schema.go @@ -24,7 +24,6 @@ func (f FieldSpec) JSONSchema() any { spec["is_deprecated"] = f.IsDeprecated spec["is_optional"] = f.IsOptional spec["is_secret"] = f.IsSecret - spec["version"] = f.Version switch f.Kind { case Kind2DArray: innerField := f diff --git a/internal/template/config.go b/internal/template/config.go index bba414ca4..18934f371 100644 --- a/internal/template/config.go +++ b/internal/template/config.go @@ -322,6 +322,7 @@ func ConfigSpec() docs.FieldSpecs { "stable", "This template is stable and will therefore not change in a breaking way outside of major version releases.", "beta", "This template is beta and will therefore not change in a breaking way unless a major problem is found.", "experimental", "This template is experimental and therefore subject to breaking changes outside of major version releases.", + "deprecated", "This template has been deprecated and should no longer be used.", ).HasDefault("stable"), docs.FieldString( "categories", "An optional list of tags, which are used for arbitrarily grouping components in documentation.", diff --git a/internal/template/template_test.go b/internal/template/template_test.go index 3c412f1e3..17d2d5dc3 100644 --- a/internal/template/template_test.go +++ b/internal/template/template_test.go @@ -671,3 +671,44 @@ mapping: | }) } } + +func TestDeprecatedTemplate(t *testing.T) { + mgr, err := manager.New(manager.NewResourceConfig()) + require.NoError(t, err) + + templateBytes := []byte(` +name: foo_memory +type: cache +status: deprecated + +fields: + - name: foovalue + type: string + +mapping: | + root.memory.init_values.foo = this.foovalue +`) + + tplConf, lints, err := template.ReadConfigYAML(mgr.Environment(), templateBytes) + require.NoError(t, err) + + assert.Empty(t, lints) + assert.Equal(t, tplConf.Status, string(docs.StatusDeprecated)) + + require.NoError(t, template.RegisterTemplateYAML(mgr.Environment(), mgr.BloblEnvironment(), templateBytes)) + + conf, err := cache.FromAny(mgr, map[string]any{ + "foo_memory": map[string]any{ + "foovalue": "meow", + }, + }) + require.NoError(t, err) + + c, err := mgr.NewCache(conf) + require.NoError(t, err) + + res, err := c.Get(t.Context(), "foo") + require.NoError(t, err) + + assert.Equal(t, "meow", string(res)) +} From c2c77f6447b697e9656a003c2d302d6a07c914ad Mon Sep 17 00:00:00 2001 From: Michal Matczuk Date: Thu, 23 Oct 2025 16:51:07 +0200 Subject: [PATCH 13/51] Network utils improvements (#297) * utils/netutil: ingest netclient * utils/netutil: refactoring * Prepare package to host more network tools * Improve spec descriptions * Refactor to decorator pattern, and support wrapping existing control functions * Detect linux based on GOOS vs build tags --- public/netclient/client_linux.go | 38 ----- public/netclient/client_other.go | 22 --- public/netclient/config.go | 94 ---------- public/netclient/spec.go | 73 -------- public/utils/netutil/dial.go | 161 ++++++++++++++++++ .../netutil/dial_test.go} | 30 +--- 6 files changed, 170 insertions(+), 248 deletions(-) delete mode 100644 public/netclient/client_linux.go delete mode 100644 public/netclient/client_other.go delete mode 100644 public/netclient/config.go delete mode 100644 public/netclient/spec.go create mode 100644 public/utils/netutil/dial.go rename public/{netclient/config_test.go => utils/netutil/dial_test.go} (61%) diff --git a/public/netclient/client_linux.go b/public/netclient/client_linux.go deleted file mode 100644 index 3dff2e1fb..000000000 --- a/public/netclient/client_linux.go +++ /dev/null @@ -1,38 +0,0 @@ -//go:build linux - -// Copyright 2025 Redpanda Data, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package netclient - -import ( - "fmt" - "syscall" -) - -// tcpUserTimeout is a linux specific constant that is used to reference -// the tcp_user_timeout option. -const tcpUserTimeout = 18 - -// SetTCPUserTimeout sets the "TCP_USER_TIMEOUT" socket option on Linux. -func (c *Config) setTCPUserTimeout(fd int) error { - timeoutMs := int(c.TCPUserTimeout.Milliseconds()) - - err := syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, tcpUserTimeout, timeoutMs) - if err != nil { - return fmt.Errorf("failed to set tcp_user_timeout: %w", err) - } - - return nil -} diff --git a/public/netclient/client_other.go b/public/netclient/client_other.go deleted file mode 100644 index 9cd42f8de..000000000 --- a/public/netclient/client_other.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build !linux - -// Copyright 2025 Redpanda Data, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package netclient - -// SetTCPUserTimeout does not apply to non-linux systems as it is not -// supported on other platforms. Errors and warnings will be silently ignored. -func (c *Config) setTCPUserTimeout(_ int) error { - return nil -} diff --git a/public/netclient/config.go b/public/netclient/config.go deleted file mode 100644 index c943179e9..000000000 --- a/public/netclient/config.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2025 Redpanda Data, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package netclient - -import ( - "fmt" - "net" - "syscall" - "time" -) - -// Config contains TCP socket configuration options. -// TCPUserTimeout is only supported on Linux -// since 2.6.37 (https://www.man7.org/linux/man-pages/man7/tcp.7.html). -// On other platforms it is ignored. -type Config struct { - KeepAliveConfig net.KeepAliveConfig - TCPUserTimeout time.Duration -} - -// Validate checks that the configuration is valid. -func (config Config) Validate() error { - // KeepAlive MUST be greater than TCP_USER_TIMEOUT - // per RFC 5482 (https://www.rfc-editor.org/rfc/rfc5482.html). - if config.TCPUserTimeout > 0 && config.KeepAliveConfig.Idle <= config.TCPUserTimeout { - return fmt.Errorf("keep_alive.idle (%s) must be greater than tcp_user_timeout (%s)", config.KeepAliveConfig.Idle, config.TCPUserTimeout) - } - return nil -} - -// NewDialerFrom creates a new net.Dialer from the provided Config. -// It validates the Config and returns an error if validation fails. -// The returned Dialer will have TCP options applied according to the Config. -func NewDialerFrom(config Config) (*net.Dialer, error) { - if err := config.Validate(); err != nil { - return nil, err - } - return config.newDialer(), nil -} - -// newDialer returns a net.Dialer configured with the TCP options from the -// Config. -func (config Config) newDialer() *net.Dialer { - dialer := &net.Dialer{ - KeepAliveConfig: config.KeepAliveConfig, - } - - if controlFn := config.controlFunc(); controlFn != nil { - dialer.Control = controlFn - } - return dialer -} - -// controlFunc returns a function that configures TCP socket options. -func (config Config) controlFunc() func(network, address string, con syscall.RawConn) error { - // Do not do anything if tcp_user_timeout is not set. - if config.TCPUserTimeout <= 0 { - return nil - } - return func(network, address string, conn syscall.RawConn) error { - var setErr error - // Starting connection to the specific file descriptor. - err := conn.Control(func(fd uintptr) { - // Set timeout. - if err := config.setTCPUserTimeout(int(fd)); err != nil { - setErr = err - return - } - }) - if err != nil { - // If no connection was able to be established then return error and - // what network and address it is trying to connect to. - return fmt.Errorf("failed to access raw connection for: %s %s: %w", network, address, err) - } - if setErr != nil { - // If connection was established, but we were unable to set the timeout - // for some reason. - return fmt.Errorf("failed to set TCP_USER_TIMEOUT (%v) on %s %s: %w", config.TCPUserTimeout, network, address, setErr) - } - return nil - } -} diff --git a/public/netclient/spec.go b/public/netclient/spec.go deleted file mode 100644 index f0a1b3e2d..000000000 --- a/public/netclient/spec.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2025 Redpanda Data, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package netclient - -import "github.com/redpanda-data/benthos/v4/public/service" - -// ConfigSpec returns the config spec for TCP options. -func ConfigSpec() *service.ConfigField { - return service.NewObjectField("tcp", - service.NewObjectField("keep_alive", - service.NewDurationField("idle"). - Description("Idle is the time that the connection must be idle before the first keep-alive probe is sent. If zero, a default value of 15 seconds is used. If negative, then keep-alive probes are disabled."). - Default("15s"), - service.NewDurationField("interval"). - Description("Interval is the time between keep-alive probes. If zero, a default value of 15 seconds is used."). - Default("15s"), - service.NewIntField("count"). - Description("Count is the maximum number of keep-alive probes that can go unanswered before dropping a connection. If zero, a default value of 9 is used"). - Default(9), - ).Description("Custom TCP keep-alive probe configuration."). - Optional(), - service.NewDurationField("tcp_user_timeout"). - Description("Linux-specific TCP_USER_TIMEOUT defines how long to wait for acknowledgment of transmitted data on an established connection before killing the connection. This allows more fine grained control on the application level as opposed to the system-wide kernel setting, tcp_retries2. If enabled, keep_alive.idle must be set to a greater value. Set to 0 (default) to disable."). - Default("0s"), - ). - Description("TCP socket configuration options"). - Optional() -} - -// ParseConfig parses a namespaced TCP config. -func ParseConfig(pConf *service.ParsedConfig) (Config, error) { - cfg := Config{} - var err error - - cfg.TCPUserTimeout, err = pConf.FieldDuration("tcp_user_timeout") - if err != nil { - return cfg, err - } - - if pConf.Contains("keep_alive") { - pc := pConf.Namespace("keep_alive") - // Each field is optional, so ignoring errors. - if idle, err := pc.FieldDuration("keep_alive"); err == nil { - cfg.KeepAliveConfig.Idle = idle - } - if interval, err := pc.FieldDuration("interval"); err == nil { - cfg.KeepAliveConfig.Interval = interval - } - if count, err := pc.FieldInt("count"); err == nil { - cfg.KeepAliveConfig.Count = count - } - // If KeepAliveConfig.Idle is a negative number then we assume they want - // KeepAlives disabled as outlined in the idle description. - if cfg.KeepAliveConfig.Idle >= 0 { - cfg.KeepAliveConfig.Enable = true - } else { - cfg.KeepAliveConfig.Enable = false - } - } - return cfg, nil -} diff --git a/public/utils/netutil/dial.go b/public/utils/netutil/dial.go new file mode 100644 index 000000000..59baafa7a --- /dev/null +++ b/public/utils/netutil/dial.go @@ -0,0 +1,161 @@ +// Copyright 2025 Redpanda Data, Inc. + +package netutil + +import ( + "context" + "fmt" + "net" + "runtime" + "syscall" + "time" + + "github.com/redpanda-data/benthos/v4/public/service" +) + +// DialerConfigSpec returns the config spec for DialerConfig. +func DialerConfigSpec() *service.ConfigField { + return service.NewObjectField("tcp", + service.NewObjectField("keep_alive", + service.NewDurationField("idle"). + Description("Duration the connection must be idle before sending the first keep-alive probe. "+ + "Zero defaults to 15s. Negative values disable keep-alive probes."). + Default("15s"), + service.NewDurationField("interval"). + Description("Duration between keep-alive probes. Zero defaults to 15s."). + Default("15s"), + service.NewIntField("count"). + Description("Maximum unanswered keep-alive probes before dropping the connection. Zero defaults to 9."). + Default(9), + ).Description("TCP keep-alive probe configuration."). + Optional(), + service.NewDurationField("tcp_user_timeout"). + Description("Maximum time to wait for acknowledgment of transmitted data before killing the connection. "+ + "Linux-only (kernel 2.6.37+), ignored on other platforms. "+ + "When enabled, keep_alive.idle must be greater than this value per RFC 5482. Zero disables."). + Default("0s"), + ). + Description("TCP socket configuration."). + Optional() +} + +// DialerConfig contains TCP socket configuration options used to configure +// a net.Dialer. +type DialerConfig struct { + KeepAliveConfig net.KeepAliveConfig + // TCPUserTimeout is only supported on Linux since 2.6.37, on other + // platforms it's ignored. + // See: https://www.man7.org/linux/man-pages/man7/tcp.7.html. + TCPUserTimeout time.Duration +} + +// DialerConfigFromParsed creates a DialerConfig from a parsed config. +func DialerConfigFromParsed(pConf *service.ParsedConfig) (DialerConfig, error) { + var ( + conf DialerConfig + err error + ) + + conf.TCPUserTimeout, err = pConf.FieldDuration("tcp_user_timeout") + if err != nil { + return conf, err + } + + if pConf.Contains("keep_alive") { + pc := pConf.Namespace("keep_alive") + + conf.KeepAliveConfig.Idle, err = pc.FieldDuration("keep_alive") + if err != nil { + return conf, err + } + + conf.KeepAliveConfig.Interval, err = pc.FieldDuration("interval") + if err != nil { + return conf, err + } + + conf.KeepAliveConfig.Count, err = pc.FieldInt("count") + if err != nil { + return conf, err + } + + // If KeepAliveConfig.Idle is a negative number then we assume they want + // KeepAlives disabled as outlined in the idle description. + if conf.KeepAliveConfig.Idle >= 0 { + conf.KeepAliveConfig.Enable = true + } else { + conf.KeepAliveConfig.Enable = false + } + } + + return conf, nil +} + +// Validate checks that the configuration is valid. +func (c DialerConfig) Validate() error { + // KeepAlive MUST be greater than TCP_USER_TIMEOUT per RFC 5482. + // See: https://www.rfc-editor.org/rfc/rfc5482.html + if c.TCPUserTimeout > 0 && c.KeepAliveConfig.Idle <= c.TCPUserTimeout { + return fmt.Errorf("keep_alive.idle (%s) must be greater than tcp_user_timeout (%s)", c.KeepAliveConfig.Idle, c.TCPUserTimeout) + } + return nil +} + +type controlContextFunc func(ctx context.Context, network, address string, conn syscall.RawConn) error + +// DecorateDialer applies DialerConfig to a net.Dialer, configuring keep-alive +// and TCP socket options. +func DecorateDialer(d *net.Dialer, conf DialerConfig) error { + if err := conf.Validate(); err != nil { + return err + } + + d.KeepAliveConfig = conf.KeepAliveConfig + + fn := d.ControlContext + if fn == nil && d.Control != nil { + fn = func(ctx context.Context, network, address string, conn syscall.RawConn) error { + return d.Control(network, address, conn) + } + } + d.Control = nil + d.ControlContext = wrapControlContext(fn, conf) + + return nil +} + +// controlFunc returns a function that configures TCP socket options. +func wrapControlContext(inner controlContextFunc, conf DialerConfig) controlContextFunc { + // We only need to wrap the control function if we have a TCPUserTimeout. + if !isLinux() || conf.TCPUserTimeout <= 0 { + return inner + } + return func(ctx context.Context, network, address string, conn syscall.RawConn) error { + if inner != nil { + if err := inner(ctx, network, address, conn); err != nil { + return err + } + } + + // tcpUserTimeout is a linux specific constant that is used to reference + // the tcp_user_timeout option. + const tcpUserTimeout = 18 + + var syscallErr error + if err := conn.Control(func(fd uintptr) { + syscallErr = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, + tcpUserTimeout, int(conf.TCPUserTimeout.Milliseconds())) + }); err != nil { + return fmt.Errorf("failed to set tcp_user_timeout: %w", err) + } + if syscallErr != nil { + return fmt.Errorf("failed to set tcp_user_timeout: %w", syscallErr) + } + + return nil + } +} + +func isLinux() bool { + return runtime.GOOS == "linux" +} diff --git a/public/netclient/config_test.go b/public/utils/netutil/dial_test.go similarity index 61% rename from public/netclient/config_test.go rename to public/utils/netutil/dial_test.go index 881d17fca..759fe20e6 100644 --- a/public/netclient/config_test.go +++ b/public/utils/netutil/dial_test.go @@ -1,18 +1,6 @@ // Copyright 2025 Redpanda Data, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package netclient +package netutil import ( "net" @@ -20,15 +8,15 @@ import ( "time" ) -func TestValidate(t *testing.T) { +func TestDialerConfigValidate(t *testing.T) { tests := []struct { name string - config Config + config DialerConfig wantErr bool }{ { name: "No TCPUserTimeout", - config: Config{ + config: DialerConfig{ TCPUserTimeout: 0, KeepAliveConfig: net.KeepAliveConfig{ Idle: 10 * time.Second, @@ -39,7 +27,7 @@ func TestValidate(t *testing.T) { { // Default value is 15seconds. name: "TCPUserTimeout set, but KeepAlive idle not set", - config: Config{ + config: DialerConfig{ TCPUserTimeout: 10 * time.Second, KeepAliveConfig: net.KeepAliveConfig{ Idle: 15 * time.Second, @@ -49,7 +37,7 @@ func TestValidate(t *testing.T) { }, { name: "KeepAlive idle less than TCPUserTimeout", - config: Config{ + config: DialerConfig{ TCPUserTimeout: 10 * time.Second, KeepAliveConfig: net.KeepAliveConfig{ Idle: 5 * time.Second, @@ -59,7 +47,7 @@ func TestValidate(t *testing.T) { }, { name: "KeepAlive idle equal to TCPUserTimeout", - config: Config{ + config: DialerConfig{ TCPUserTimeout: 10 * time.Second, KeepAliveConfig: net.KeepAliveConfig{ Idle: 10 * time.Second, @@ -69,7 +57,7 @@ func TestValidate(t *testing.T) { }, { name: "KeepAlive idle greater than TCPUserTimeout", - config: Config{ + config: DialerConfig{ TCPUserTimeout: 10 * time.Second, KeepAliveConfig: net.KeepAliveConfig{ Idle: 30 * time.Second, @@ -82,7 +70,7 @@ func TestValidate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if err := tt.config.Validate(); (err != nil) != tt.wantErr { - t.Errorf("Config.Validate() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("DialerConfig.Validate() error = %v, wantErr %v", err, tt.wantErr) } }) } From a2e715d6d18fe52c542412901ee6a59c5c18e68e Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Thu, 23 Oct 2025 16:04:38 +0100 Subject: [PATCH 14/51] Update CL (#303) --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d21a95edc..72747903f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,17 @@ Changelog All notable changes to this project will be documented in this file. +## 4.58.0 - 2025-10-23 + +### Added + +- New `public/utils/netutil` package. (@alextreichler) +- Exporting a schema with the format `jsonschema` now includes `is_advanced`, `is_deprecated`, `is_optional`, `is_secret` extra fields. (@tomasz-sadura) + +### Fixed + +- Templates are now able to mark themselves as deprecated. (@Jeffail) + ## 4.57.1 - 2025-10-02 ### Fixed From 76bf934cf15e43041b0c9a0ba228e1e162f5eecd Mon Sep 17 00:00:00 2001 From: Tyler Rockwood Date: Wed, 29 Oct 2025 22:07:46 -0500 Subject: [PATCH 15/51] bloblang: add "".repeat(N) method (#305) * bloblang: add "".repeat(N) method * update changelog --- CHANGELOG.md | 6 +++ internal/bloblang/query/methods_strings.go | 45 ++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72747903f..9ba156f24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ Changelog All notable changes to this project will be documented in this file. +## 4.59.0 - TBD + +### Added + +- New `string.repeat(int)` method to repeat a string or byte array N times. (@rockwotj) + ## 4.58.0 - 2025-10-23 ### Added diff --git a/internal/bloblang/query/methods_strings.go b/internal/bloblang/query/methods_strings.go index 627d4e35d..650163e24 100644 --- a/internal/bloblang/query/methods_strings.go +++ b/internal/bloblang/query/methods_strings.go @@ -23,6 +23,8 @@ import ( "hash/fnv" "html" "io" + "math" + "math/bits" "net/url" "path/filepath" "regexp" @@ -2119,3 +2121,46 @@ root.description = this.description.trim_suffix("_foobar")`, }, nil }, ) + +//------------------------------------------------------------------------------ + +var _ = registerSimpleMethod( + NewMethodSpec( + "repeat", "", + ).InCategory( + MethodCategoryStrings, + "Repeat returns a new string consisting of count copies of the string", + NewExampleSpec("", + `root.repeated = this.name.repeat(3) +root.not_repeated = this.name.repeat(0)`, + `{"name":"bob"}`, + `{"not_repeated":"","repeated":"bobbobbob"}`, + ), + ).Param(ParamInt64("count", "The number of times to repeat the string.")), + func(args *ParsedParams) (simpleMethod, error) { + count, err := args.FieldInt64("count") + if err != nil { + return nil, err + } + if count < 0 { + return nil, fmt.Errorf("invalid count, must be greater than or equal to zero: %d", count) + } + return func(v any, ctx FunctionContext) (any, error) { + switch t := v.(type) { + case string: + hi, lo := bits.Mul(uint(len(t)), uint(count)) + if hi > 0 || lo > uint(math.MaxInt) { + return nil, fmt.Errorf("invalid count, would overflow: %d*%d", len(t), count) + } + return strings.Repeat(t, int(count)), nil + case []byte: + hi, lo := bits.Mul(uint(len(t)), uint(count)) + if hi > 0 || lo > uint(math.MaxInt) { + return nil, fmt.Errorf("invalid count, would overflow: %d*%d", len(t), count) + } + return bytes.Repeat(t, int(count)), nil + } + return nil, value.NewTypeError(v, value.TString) + }, nil + }, +) From 87fb5183a88a9b8835eb9eb8a1e482b06a454d13 Mon Sep 17 00:00:00 2001 From: Alex Treichler Date: Thu, 30 Oct 2025 07:48:55 -0700 Subject: [PATCH 16/51] netutil: fix the name of keep_alive idle value --- public/utils/netutil/dial.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/utils/netutil/dial.go b/public/utils/netutil/dial.go index 59baafa7a..0b6065fb6 100644 --- a/public/utils/netutil/dial.go +++ b/public/utils/netutil/dial.go @@ -64,7 +64,7 @@ func DialerConfigFromParsed(pConf *service.ParsedConfig) (DialerConfig, error) { if pConf.Contains("keep_alive") { pc := pConf.Namespace("keep_alive") - conf.KeepAliveConfig.Idle, err = pc.FieldDuration("keep_alive") + conf.KeepAliveConfig.Idle, err = pc.FieldDuration("idle") if err != nil { return conf, err } From ad912c65852c7222c12020298879c021c5d60487 Mon Sep 17 00:00:00 2001 From: Tyler Rockwood Date: Thu, 30 Oct 2025 10:44:09 -0500 Subject: [PATCH 17/51] bloblang: add bytes function (#308) --- CHANGELOG.md | 1 + internal/bloblang/query/functions.go | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ba156f24..5ab5bce4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. ### Added - New `string.repeat(int)` method to repeat a string or byte array N times. (@rockwotj) +- New `bytes` method to create a 0 initialized byte array. (@rockwotj) ## 4.58.0 - 2025-10-23 diff --git a/internal/bloblang/query/functions.go b/internal/bloblang/query/functions.go index 88044b008..ac883df9f 100644 --- a/internal/bloblang/query/functions.go +++ b/internal/bloblang/query/functions.go @@ -1034,3 +1034,25 @@ func NewVarFunction(name string) Function { return ctx, paths }) } + +//------------------------------------------------------------------------------ + +var _ = registerFunction( + NewFunctionSpec( + FunctionCategoryGeneral, "bytes", + "Create a new byte array that is zero initialized", + NewExampleSpec("", + `root.data = bytes(5)`, + `{"data":"AAAAAAAK"}`, + ), + ).Param(ParamInt64("length", "The size of the resulting byte array.")), + func(args *ParsedParams) (Function, error) { + length, err := args.FieldInt64("length") + if err != nil { + return nil, err + } + return ClosureFunction("function bytes", func(_ FunctionContext) (any, error) { + return make([]byte, length), nil + }, nil), nil + }, +) From 44aea248730d53ef67b0746ab359a15982fc83fc Mon Sep 17 00:00:00 2001 From: Josh Purcell Date: Fri, 31 Oct 2025 05:58:21 -0500 Subject: [PATCH 18/51] netutil: enable decorating listener with SO_REUSEADDR and SO_REUSEPORT --- public/utils/netutil/listen.go | 73 ++++++++++++++ public/utils/netutil/listen_test.go | 148 ++++++++++++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 public/utils/netutil/listen.go create mode 100644 public/utils/netutil/listen_test.go diff --git a/public/utils/netutil/listen.go b/public/utils/netutil/listen.go new file mode 100644 index 000000000..86e5f2be9 --- /dev/null +++ b/public/utils/netutil/listen.go @@ -0,0 +1,73 @@ +// Copyright 2025 Redpanda Data, Inc. + +package netutil + +import ( + "fmt" + "net" + "syscall" +) + +// ListenerConfig contains TCP listener socket configuration options. +type ListenerConfig struct { + // ReuseAddr enables SO_REUSEADDR, allowing binding to ports in TIME_WAIT state. + // Useful for graceful restarts and config reloads where the server needs to + // rebind to the same port immediately after shutdown. + ReuseAddr bool + + // ReusePort enables SO_REUSEPORT, allowing multiple sockets to bind to the same + // port for load balancing across multiple processes/threads. + ReusePort bool +} + +// DecorateListenerConfig applies ListenerConfig settings to a net.ListenConfig. +// This configures socket options like SO_REUSEADDR and SO_REUSEPORT. +func DecorateListenerConfig(lc *net.ListenConfig, conf ListenerConfig) error { + // If no options are set, nothing to do + if !conf.ReuseAddr && !conf.ReusePort { + return nil + } + + // Wrap any existing Control function + existingControl := lc.Control + lc.Control = func(network, address string, c syscall.RawConn) error { + // Call existing control function first if it exists + if existingControl != nil { + if err := existingControl(network, address, c); err != nil { + return err + } + } + + // Apply socket options + var sockOptErr error + if err := c.Control(func(fd uintptr) { + if conf.ReuseAddr { + sockOptErr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1) + if sockOptErr != nil { + return + } + } + + if conf.ReusePort { + // SO_REUSEPORT = 15 on Linux, not available on all platforms + const SO_REUSEPORT = 0x0F + sockOptErr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, SO_REUSEPORT, 1) + if sockOptErr != nil { + // Ignore error if SO_REUSEPORT is not supported on this platform + // This allows the code to work across different OSes + sockOptErr = nil + } + } + }); err != nil { + return fmt.Errorf("failed to access raw socket connection: %w", err) + } + + if sockOptErr != nil { + return fmt.Errorf("failed to set socket options: %w", sockOptErr) + } + + return nil + } + + return nil +} diff --git a/public/utils/netutil/listen_test.go b/public/utils/netutil/listen_test.go new file mode 100644 index 000000000..383412a86 --- /dev/null +++ b/public/utils/netutil/listen_test.go @@ -0,0 +1,148 @@ +// Copyright 2025 Redpanda Data, Inc. + +package netutil + +import ( + "context" + "net" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestDecorateListenConfig(t *testing.T) { + ctx := context.Background() + + // Test decorating an existing ListenConfig + lc := net.ListenConfig{} + conf := ListenerConfig{ + ReuseAddr: true, + } + + err := DecorateListenerConfig(&lc, conf) + require.NoError(t, err) + + listener1, err := lc.Listen(ctx, "tcp", "127.0.0.1:0") + require.NoError(t, err) + defer listener1.Close() + + addr := listener1.Addr().String() + require.NoError(t, listener1.Close()) + + time.Sleep(10 * time.Millisecond) + + // Should be able to rebind immediately with decorated config + listener2, err := lc.Listen(ctx, "tcp", addr) + require.NoError(t, err, "Failed to bind with decorated ListenConfig") + defer listener2.Close() +} + +func TestListenerConfig_ServerReload(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Use a fixed port for this test + addr := "127.0.0.1:19284" + + conf := ListenerConfig{ + ReuseAddr: true, + } + + // Create first server + lc1 := net.ListenConfig{} + err := DecorateListenerConfig(&lc1, conf) + require.NoError(t, err) + + listener1, err := lc1.Listen(ctx, "tcp", addr) + require.NoError(t, err) + + server1 := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("server1")) + }), + } + + serverErr := make(chan error, 1) + go func() { + err := server1.Serve(listener1) + if err != nil && err != http.ErrServerClosed { + serverErr <- err + } + close(serverErr) + }() + + // Give server time to start + time.Sleep(100 * time.Millisecond) + + // Make a request to ensure server is working + resp, err := http.Get("http://" + addr) + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + // Shutdown first server + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownCancel() + require.NoError(t, server1.Shutdown(shutdownCtx)) + + // Wait for server to fully stop + select { + case err := <-serverErr: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("Server did not stop in time") + } + + // Small delay to ensure port is released + time.Sleep(100 * time.Millisecond) + + // Create second server on same address - should succeed due to SO_REUSEADDR + lc2 := net.ListenConfig{} + err = DecorateListenerConfig(&lc2, conf) + require.NoError(t, err) + + listener2, err := lc2.Listen(ctx, "tcp", addr) + require.NoError(t, err, "Failed to bind to port after server shutdown - SO_REUSEADDR may not be working") + + server2 := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("server2")) + }), + } + + go func() { + _ = server2.Serve(listener2) + }() + + // Give server time to start + time.Sleep(100 * time.Millisecond) + + // Make a request to ensure new server is working + resp, err = http.Get("http://" + addr) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + // Cleanup + shutdownCtx2, shutdownCancel2 := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownCancel2() + require.NoError(t, server2.Shutdown(shutdownCtx2)) +} + +func TestListenerConfig_Empty(t *testing.T) { + // Test that empty config doesn't break anything + lc := net.ListenConfig{} + conf := ListenerConfig{} + + err := DecorateListenerConfig(&lc, conf) + require.NoError(t, err) + + // Should still be able to listen normally + listener, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() +} From cea8d610b69c7b91c2a382a0cf14d63c71e446cc Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Mon, 3 Nov 2025 14:20:01 +0000 Subject: [PATCH 19/51] Update CL (#311) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ab5bce4c..dd9dbb1c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ Changelog All notable changes to this project will be documented in this file. -## 4.59.0 - TBD +## 4.59.0 - 2025-11-03 ### Added From c8fd7bc40ef6b189310dbdc20849325e474bc59d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Tue, 4 Nov 2025 14:48:24 +0100 Subject: [PATCH 20/51] netutil: add connect_timeout to DialerConfig Make net.Dialer.Timeout configurable. --- CHANGELOG.md | 6 ++++++ public/utils/netutil/dial.go | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd9dbb1c3..adc998203 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ Changelog All notable changes to this project will be documented in this file. +## 4.60.0 - TBD + +### Added + +- New `connect_timeout` to `tcp` dialer config. (@mmatczuk) + ## 4.59.0 - 2025-11-03 ### Added diff --git a/public/utils/netutil/dial.go b/public/utils/netutil/dial.go index 0b6065fb6..ea94e9cbb 100644 --- a/public/utils/netutil/dial.go +++ b/public/utils/netutil/dial.go @@ -16,6 +16,9 @@ import ( // DialerConfigSpec returns the config spec for DialerConfig. func DialerConfigSpec() *service.ConfigField { return service.NewObjectField("tcp", + service.NewDurationField("connect_timeout"). + Description("Maximum amount of time a dial will wait for a connect to complete. Zero disables."). + Default("0s"), service.NewObjectField("keep_alive", service.NewDurationField("idle"). Description("Duration the connection must be idle before sending the first keep-alive probe. "+ @@ -42,6 +45,11 @@ func DialerConfigSpec() *service.ConfigField { // DialerConfig contains TCP socket configuration options used to configure // a net.Dialer. type DialerConfig struct { + // Timeout is the maximum amount of time a dial will wait for a connect to + // complete. If Deadline is also set, it may fail earlier. + // + // The default is no timeout. + Timeout time.Duration KeepAliveConfig net.KeepAliveConfig // TCPUserTimeout is only supported on Linux since 2.6.37, on other // platforms it's ignored. @@ -56,6 +64,11 @@ func DialerConfigFromParsed(pConf *service.ParsedConfig) (DialerConfig, error) { err error ) + conf.Timeout, err = pConf.FieldDuration("connect_timeout") + if err != nil { + return conf, err + } + conf.TCPUserTimeout, err = pConf.FieldDuration("tcp_user_timeout") if err != nil { return conf, err @@ -110,6 +123,7 @@ func DecorateDialer(d *net.Dialer, conf DialerConfig) error { return err } + d.Timeout = conf.Timeout d.KeepAliveConfig = conf.KeepAliveConfig fn := d.ControlContext From e6717eda1acd7a7c9babfa40348124175f7ceb7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Tue, 4 Nov 2025 14:48:24 +0100 Subject: [PATCH 21/51] netutil: add connect_timeout to DialerConfig --- public/utils/netutil/dial.go | 1 + 1 file changed, 1 insertion(+) diff --git a/public/utils/netutil/dial.go b/public/utils/netutil/dial.go index ea94e9cbb..7945108a7 100644 --- a/public/utils/netutil/dial.go +++ b/public/utils/netutil/dial.go @@ -125,6 +125,7 @@ func DecorateDialer(d *net.Dialer, conf DialerConfig) error { d.Timeout = conf.Timeout d.KeepAliveConfig = conf.KeepAliveConfig + d.Timeout = conf.Timeout fn := d.ControlContext if fn == nil && d.Control != nil { From a7dda98cb25a96822dac6952d14cb58beb7a3e75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Tue, 4 Nov 2025 15:07:48 +0100 Subject: [PATCH 22/51] netutil: make DialerConfigSpec() return advanced by default --- public/utils/netutil/dial.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/public/utils/netutil/dial.go b/public/utils/netutil/dial.go index 7945108a7..905a394d5 100644 --- a/public/utils/netutil/dial.go +++ b/public/utils/netutil/dial.go @@ -39,7 +39,8 @@ func DialerConfigSpec() *service.ConfigField { Default("0s"), ). Description("TCP socket configuration."). - Optional() + Optional(). + Advanced() } // DialerConfig contains TCP socket configuration options used to configure From 80f3b69bc280aa5e8f6171cc37cbf21274208cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Tue, 4 Nov 2025 15:12:55 +0100 Subject: [PATCH 23/51] netutil: add ListenerConfigSpec() and ListenerConfig constructor from parsed config --- public/utils/netutil/listen.go | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/public/utils/netutil/listen.go b/public/utils/netutil/listen.go index 86e5f2be9..14e5a0ba9 100644 --- a/public/utils/netutil/listen.go +++ b/public/utils/netutil/listen.go @@ -6,8 +6,28 @@ import ( "fmt" "net" "syscall" + + "github.com/redpanda-data/benthos/v4/public/service" ) +// ListenerConfigSpec returns the config spec for ListenerConfig. +func ListenerConfigSpec() *service.ConfigField { + return service.NewObjectField("tcp", + service.NewBoolField("reuse_addr"). + Description("Enable SO_REUSEADDR, allowing binding to ports in TIME_WAIT state. "+ + "Useful for graceful restarts and config reloads where the server needs to "+ + "rebind to the same port immediately after shutdown."). + Default(false), + service.NewBoolField("reuse_port"). + Description("Enable SO_REUSEPORT, allowing multiple sockets to bind to the same "+ + "port for load balancing across multiple processes/threads."). + Default(false), + ). + Description("TCP listener socket configuration."). + Optional(). + Advanced() +} + // ListenerConfig contains TCP listener socket configuration options. type ListenerConfig struct { // ReuseAddr enables SO_REUSEADDR, allowing binding to ports in TIME_WAIT state. @@ -20,6 +40,26 @@ type ListenerConfig struct { ReusePort bool } +// ListenerConfigFromParsed creates a ListenerConfig from a parsed config. +func ListenerConfigFromParsed(pConf *service.ParsedConfig) (ListenerConfig, error) { + var ( + conf ListenerConfig + err error + ) + + conf.ReuseAddr, err = pConf.FieldBool("reuse_addr") + if err != nil { + return conf, err + } + + conf.ReusePort, err = pConf.FieldBool("reuse_port") + if err != nil { + return conf, err + } + + return conf, nil +} + // DecorateListenerConfig applies ListenerConfig settings to a net.ListenConfig. // This configures socket options like SO_REUSEADDR and SO_REUSEPORT. func DecorateListenerConfig(lc *net.ListenConfig, conf ListenerConfig) error { From bad4c4ca1981474fa00fe3ef122561a39ea62af8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Thu, 6 Nov 2025 10:12:11 +0100 Subject: [PATCH 24/51] netutil: add support for windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without this patch Windows build fail with error= │ build failed: exit status 1: # github.com/redpanda-data/benthos/v4/public/utils/netutil │ ../../../Go/pkg/mod/github.com/redpanda-data/benthos/v4@v4.59.0/public/utils/netutil/dial.go:146:39: cannot use int(fd) (value of type int) as syscall.Handle value in argument to syscall.SetsockoptInt │ ../../../Go/pkg/mod/github.com/redpanda-data/benthos/v4@v4.59.0/public/utils/netutil/listen.go:45:40: cannot use int(fd) (value of type int) as syscall.Handle value in argument to syscall.SetsockoptInt │ ../../../Go/pkg/mod/github.com/redpanda-data/benthos/v4@v4.59.0/public/utils/netutil/listen.go:54:40: cannot use int(fd) (value of type int) as syscall.Handle value in argument to syscall.SetsockoptInt target=windows_amd64_v1 --- public/utils/netutil/dial.go | 2 +- public/utils/netutil/listen.go | 4 ++-- public/utils/netutil/syscall_others.go | 12 ++++++++++++ public/utils/netutil/syscall_windows.go | 12 ++++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 public/utils/netutil/syscall_others.go create mode 100644 public/utils/netutil/syscall_windows.go diff --git a/public/utils/netutil/dial.go b/public/utils/netutil/dial.go index 905a394d5..8a05555a8 100644 --- a/public/utils/netutil/dial.go +++ b/public/utils/netutil/dial.go @@ -159,7 +159,7 @@ func wrapControlContext(inner controlContextFunc, conf DialerConfig) controlCont var syscallErr error if err := conn.Control(func(fd uintptr) { - syscallErr = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, + syscallErr = setsockoptInt(fd, syscall.IPPROTO_TCP, tcpUserTimeout, int(conf.TCPUserTimeout.Milliseconds())) }); err != nil { return fmt.Errorf("failed to set tcp_user_timeout: %w", err) diff --git a/public/utils/netutil/listen.go b/public/utils/netutil/listen.go index 14e5a0ba9..e1b76de8c 100644 --- a/public/utils/netutil/listen.go +++ b/public/utils/netutil/listen.go @@ -82,7 +82,7 @@ func DecorateListenerConfig(lc *net.ListenConfig, conf ListenerConfig) error { var sockOptErr error if err := c.Control(func(fd uintptr) { if conf.ReuseAddr { - sockOptErr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1) + sockOptErr = setsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1) if sockOptErr != nil { return } @@ -91,7 +91,7 @@ func DecorateListenerConfig(lc *net.ListenConfig, conf ListenerConfig) error { if conf.ReusePort { // SO_REUSEPORT = 15 on Linux, not available on all platforms const SO_REUSEPORT = 0x0F - sockOptErr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, SO_REUSEPORT, 1) + sockOptErr = setsockoptInt(fd, syscall.SOL_SOCKET, SO_REUSEPORT, 1) if sockOptErr != nil { // Ignore error if SO_REUSEPORT is not supported on this platform // This allows the code to work across different OSes diff --git a/public/utils/netutil/syscall_others.go b/public/utils/netutil/syscall_others.go new file mode 100644 index 000000000..95c76e74e --- /dev/null +++ b/public/utils/netutil/syscall_others.go @@ -0,0 +1,12 @@ +//go:build !windows + +// Copyright 2025 Redpanda Data, Inc. + +package netutil + +import "syscall" + +// setsockoptInt wraps syscall.SetsockoptInt for Unix-like systems. +func setsockoptInt(fd uintptr, level, opt, value int) error { + return syscall.SetsockoptInt(int(fd), level, opt, value) +} diff --git a/public/utils/netutil/syscall_windows.go b/public/utils/netutil/syscall_windows.go new file mode 100644 index 000000000..55718698f --- /dev/null +++ b/public/utils/netutil/syscall_windows.go @@ -0,0 +1,12 @@ +//go:build windows + +// Copyright 2025 Redpanda Data, Inc. + +package netutil + +import "syscall" + +// setsockoptInt wraps syscall.SetsockoptInt for Windows. +func setsockoptInt(fd uintptr, level, opt, value int) error { + return syscall.SetsockoptInt(syscall.Handle(fd), level, opt, value) +} From 05de4227e256f2741307dc32321869f012d8309b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Fri, 7 Nov 2025 10:40:48 +0100 Subject: [PATCH 25/51] Update CL --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adc998203..b86dd5629 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,16 @@ Changelog All notable changes to this project will be documented in this file. -## 4.60.0 - TBD +## 4.60.0 - 2025-11-07 ### Added - New `connect_timeout` to `tcp` dialer config. (@mmatczuk) +### Fixed + +- Fixed build failure on Windows caused by syscall signature miss-match in `netutil`. (@mmatczuk) + ## 4.59.0 - 2025-11-03 ### Added From 88db1772a5a3a4d6e2e0aa83907e1c99646be68e Mon Sep 17 00:00:00 2001 From: Alex Treichler Date: Fri, 14 Nov 2025 05:42:21 -0800 Subject: [PATCH 26/51] io: add unixgram option to input socket server --- internal/impl/io/input_socket_server.go | 6 +- internal/impl/io/input_socket_server_test.go | 400 +++++++++++++++++++ internal/stream/type.go | 12 +- 3 files changed, 409 insertions(+), 9 deletions(-) diff --git a/internal/impl/io/input_socket_server.go b/internal/impl/io/input_socket_server.go index 13e119ecf..b632c3426 100644 --- a/internal/impl/io/input_socket_server.go +++ b/internal/impl/io/input_socket_server.go @@ -48,7 +48,7 @@ func socketServerInputSpec() *service.ConfigSpec { Summary(`Creates a server that receives a stream of messages over a TCP, UDP or Unix socket.`). Categories("Network"). Fields( - service.NewStringEnumField(issFieldNetwork, "unix", "tcp", "udp", "tls"). + service.NewStringEnumField(issFieldNetwork, "unix", "tcp", "udp", "tls", "unixgram"). Description("A network type to accept."), service.NewStringField(isFieldAddress). Description("The address to listen from."). @@ -182,7 +182,7 @@ func (t *socketServerInput) Connect(ctx context.Context) error { ClientAuth: t.tlsClientAuth, } ln, err = tls.Listen("tcp", t.address, config) - case "udp": + case "udp", "unixgram": cn, err = net.ListenPacket(t.network, t.address) default: return fmt.Errorf("socket network '%v' is not supported by this input", t.network) @@ -203,7 +203,7 @@ func (t *socketServerInput) Connect(ctx context.Context) error { t.log.Infof("Receiving %v socket messages from address: %v", t.network, addr.String()) } else { addr = cn.LocalAddr() - t.log.Infof("Receiving udp socket messages from address: %v", addr.String()) + t.log.Infof("Receiving %v socket messages from address: %v", t.network, addr.String()) } if t.addressCache != "" { key := "socket_server_address" diff --git a/internal/impl/io/input_socket_server_test.go b/internal/impl/io/input_socket_server_test.go index 48dc53beb..58bd4c062 100644 --- a/internal/impl/io/input_socket_server_test.go +++ b/internal/impl/io/input_socket_server_test.go @@ -495,6 +495,406 @@ socket_server: wg.Wait() } +func TestUnixgramSocketServerBasic(t *testing.T) { + ctx, done := context.WithTimeout(t.Context(), time.Second*20) + defer done() + + tmpDir := t.TempDir() + + rdr, addr := socketServerInputFromConf(t, ` +socket_server: + network: unixgram + address: %v +`, filepath.Join(tmpDir, "benthos.sock")) + + defer func() { + rdr.TriggerStopConsuming() + assert.NoError(t, rdr.WaitForClose(ctx)) + }() + + conn, err := net.Dial("unixgram", addr) + require.NoError(t, err) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + _ = conn.SetWriteDeadline(time.Now().Add(time.Second * 5)) + + _, cerr := conn.Write([]byte("foo\n")) + require.NoError(t, cerr) + + _, cerr = conn.Write([]byte("bar\n")) + require.NoError(t, cerr) + + _, cerr = conn.Write([]byte("baz\n")) + require.NoError(t, cerr) + + wg.Done() + }() + + readNextMsg := func() (message.Batch, error) { + var tran message.Transaction + select { + case tran = <-rdr.TransactionChan(): + require.NoError(t, tran.Ack(ctx, nil)) + case <-time.After(time.Second): + return nil, errors.New("timed out") + } + return tran.Payload, nil + } + + exp := [][]byte{[]byte("foo")} + msg, err := readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + exp = [][]byte{[]byte("bar")} + msg, err = readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + exp = [][]byte{[]byte("baz")} + msg, err = readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + wg.Wait() + conn.Close() +} + +func TestUnixgramSocketServerRetries(t *testing.T) { + ctx, done := context.WithTimeout(t.Context(), time.Second*20) + defer done() + + tmpDir := t.TempDir() + + rdr, addr := socketServerInputFromConf(t, ` +socket_server: + network: unixgram + address: %v +`, filepath.Join(tmpDir, "benthos.sock")) + + defer func() { + rdr.TriggerStopConsuming() + assert.NoError(t, rdr.WaitForClose(ctx)) + }() + + conn, err := net.Dial("unixgram", addr) + require.NoError(t, err) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + _ = conn.SetWriteDeadline(time.Now().Add(time.Second * 5)) + + _, cerr := conn.Write([]byte("foo\n")) + require.NoError(t, cerr) + + _, cerr = conn.Write([]byte("bar\n")) + require.NoError(t, cerr) + + _, cerr = conn.Write([]byte("baz\n")) + require.NoError(t, cerr) + + wg.Done() + }() + + readNextMsg := func(reject bool) (message.Batch, error) { + var tran message.Transaction + select { + case tran = <-rdr.TransactionChan(): + var res error + if reject { + res = errors.New("test err") + } + require.NoError(t, tran.Ack(ctx, res)) + case <-time.After(time.Second * 5): + return nil, errors.New("timed out") + } + return tran.Payload, nil + } + + exp := [][]byte{[]byte("foo")} + msg, err := readNextMsg(false) + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + exp = [][]byte{[]byte("bar")} + msg, err = readNextMsg(true) + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + expRemaining := []string{"bar", "baz"} + actRemaining := []string{} + + msg, err = readNextMsg(false) + require.NoError(t, err) + require.Equal(t, 1, msg.Len()) + actRemaining = append(actRemaining, string(msg.Get(0).AsBytes())) + + msg, err = readNextMsg(false) + require.NoError(t, err) + require.Equal(t, 1, msg.Len()) + actRemaining = append(actRemaining, string(msg.Get(0).AsBytes())) + + sort.Strings(actRemaining) + assert.Equal(t, expRemaining, actRemaining) + + wg.Wait() + conn.Close() +} + +func TestUnixgramServerWriteToClosed(t *testing.T) { + ctx, done := context.WithTimeout(t.Context(), time.Second*20) + defer done() + + tmpDir := t.TempDir() + + rdr, addr := socketServerInputFromConf(t, ` +socket_server: + network: unixgram + address: %v +`, filepath.Join(tmpDir, "benthos.sock")) + + conn, err := net.Dial("unixgram", addr) + require.NoError(t, err) + + _ = conn.SetWriteDeadline(time.Now().Add(time.Second * 5)) + + rdr.TriggerStopConsuming() + assert.NoError(t, rdr.WaitForClose(ctx)) + + // Just make sure data written doesn't panic + _, _ = conn.Write([]byte("bar\n")) + + _, open := <-rdr.TransactionChan() + assert.False(t, open) + + conn.Close() +} + +func TestUnixgramSocketServerReconnect(t *testing.T) { + ctx, done := context.WithTimeout(t.Context(), time.Second*20) + defer done() + + tmpDir := t.TempDir() + + rdr, addr := socketServerInputFromConf(t, ` +socket_server: + network: unixgram + address: %v +`, filepath.Join(tmpDir, "benthos.sock")) + + defer func() { + rdr.TriggerStopConsuming() + assert.NoError(t, rdr.WaitForClose(ctx)) + }() + + conn, err := net.Dial("unixgram", addr) + require.NoError(t, err) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + _ = conn.SetWriteDeadline(time.Now().Add(time.Second * 5)) + _, cerr := conn.Write([]byte("foo\n")) + require.NoError(t, cerr) + + conn.Close() + + conn, cerr = net.Dial("unixgram", addr) + require.NoError(t, cerr) + + _, cerr = conn.Write([]byte("bar\n")) + require.NoError(t, cerr) + + _, cerr = conn.Write([]byte("baz\n")) + require.NoError(t, cerr) + + wg.Done() + }() + + readNextMsg := func() (message.Batch, error) { + var tran message.Transaction + select { + case tran = <-rdr.TransactionChan(): + require.NoError(t, tran.Ack(ctx, nil)) + case <-time.After(time.Second): + return nil, errors.New("timed out") + } + return tran.Payload, nil + } + + exp := [][]byte{[]byte("foo")} + msg, err := readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + exp = [][]byte{[]byte("bar")} + msg, err = readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + exp = [][]byte{[]byte("baz")} + msg, err = readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + wg.Wait() + conn.Close() +} + +func TestUnixgramSocketServerCustomDelim(t *testing.T) { + ctx, done := context.WithTimeout(t.Context(), time.Second*20) + defer done() + + tmpDir := t.TempDir() + + rdr, addr := socketServerInputFromConf(t, ` +socket_server: + network: unixgram + address: %v + codec: delim:@ +`, filepath.Join(tmpDir, "benthos.sock")) + + defer func() { + rdr.TriggerStopConsuming() + assert.NoError(t, rdr.WaitForClose(ctx)) + }() + + conn, err := net.Dial("unixgram", addr) + require.NoError(t, err) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + _ = conn.SetWriteDeadline(time.Now().Add(time.Second * 5)) + + _, cerr := conn.Write([]byte("foo@")) + require.NoError(t, cerr) + + _, cerr = conn.Write([]byte("bar@")) + require.NoError(t, cerr) + + _, cerr = conn.Write([]byte("@")) + require.NoError(t, cerr) + + _, cerr = conn.Write([]byte("baz\n@@")) + require.NoError(t, cerr) + + wg.Done() + }() + + readNextMsg := func() (message.Batch, error) { + var tran message.Transaction + select { + case tran = <-rdr.TransactionChan(): + require.NoError(t, tran.Ack(ctx, nil)) + case <-time.After(time.Second): + return nil, errors.New("timed out") + } + return tran.Payload, nil + } + + exp := [][]byte{[]byte("foo")} + msg, err := readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + exp = [][]byte{[]byte("bar")} + msg, err = readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + exp = [][]byte{[]byte("")} + msg, err = readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + exp = [][]byte{[]byte("baz\n")} + msg, err = readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + wg.Wait() + conn.Close() +} + +func TestUnixgramSocketServerShutdown(t *testing.T) { + ctx, done := context.WithTimeout(t.Context(), time.Second*20) + defer done() + + tmpDir := t.TempDir() + + rdr, addr := socketServerInputFromConf(t, ` +socket_server: + network: unixgram + address: %v +`, filepath.Join(tmpDir, "benthos.sock")) + + defer func() { + rdr.TriggerStopConsuming() + assert.NoError(t, rdr.WaitForClose(ctx)) + }() + + conn, err := net.Dial("unixgram", addr) + require.NoError(t, err) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + _ = conn.SetWriteDeadline(time.Now().Add(time.Second * 5)) + + _, cerr := conn.Write([]byte("foo\n")) + require.NoError(t, cerr) + + _, cerr = conn.Write([]byte("bar\n")) + require.NoError(t, cerr) + + _, cerr = conn.Write([]byte("\n")) + require.NoError(t, cerr) + + _, cerr = conn.Write([]byte("baz\n")) + require.NoError(t, cerr) + + conn.Close() + wg.Done() + }() + + readNextMsg := func() (message.Batch, error) { + var tran message.Transaction + select { + case tran = <-rdr.TransactionChan(): + require.NoError(t, tran.Ack(ctx, nil)) + case <-time.After(time.Second): + return nil, errors.New("timed out") + } + return tran.Payload, nil + } + + exp := [][]byte{[]byte("foo")} + msg, err := readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + exp = [][]byte{[]byte("bar")} + msg, err = readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + exp = [][]byte{[]byte("")} + msg, err = readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + exp = [][]byte{[]byte("baz")} + msg, err = readNextMsg() + require.NoError(t, err) + assert.Equal(t, exp, message.GetAllBytes(msg)) + + wg.Wait() +} + func TestSocketUDPServerBasic(t *testing.T) { ctx, done := context.WithTimeout(t.Context(), time.Second*20) defer done() diff --git a/internal/stream/type.go b/internal/stream/type.go index d0d1babce..417df516f 100644 --- a/internal/stream/type.go +++ b/internal/stream/type.go @@ -182,13 +182,13 @@ func (t *Type) start() (err error) { } go func(out output.Streamed) { - for { - if err := out.WaitForClose(context.Background()); err == nil { - t.onClose() - atomic.StoreUint32(&t.closed, 1) - return - } + // WaitForClose blocks until output is fully closed. + // Errors indicate close completed with issues, but output is still closed. + if err := out.WaitForClose(context.Background()); err != nil { + t.manager.Logger().Debug("Output close completed with error: %v", err) } + t.onClose() + atomic.StoreUint32(&t.closed, 1) }(t.outputLayer) return nil From 8f6ea44662631401b3b3d963bb5c0f948b1a772d Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Mon, 17 Nov 2025 11:21:33 +0000 Subject: [PATCH 27/51] build(deps): bump shutdown dependency (#320) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c54ee6959..844bdfc1b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ require ( cuelang.org/go v0.13.2 github.com/Jeffail/gabs/v2 v2.7.0 github.com/Jeffail/grok v1.1.0 - github.com/Jeffail/shutdown v1.0.0 + github.com/Jeffail/shutdown v1.1.0 github.com/OneOfOne/xxhash v1.2.8 github.com/cenkalti/backoff/v4 v4.3.0 github.com/dustin/go-humanize v1.0.1 diff --git a/go.sum b/go.sum index 339bbdb00..49b4223f3 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/Jeffail/gabs/v2 v2.7.0 h1:Y2edYaTcE8ZpRsR2AtmPu5xQdFDIthFG0jYhu5PY8kg github.com/Jeffail/gabs/v2 v2.7.0/go.mod h1:dp5ocw1FvBBQYssgHsG7I1WYsiLRtkUaB1FEtSwvNUw= github.com/Jeffail/grok v1.1.0 h1:kiHmZ+0J5w/XUihRgU3DY9WIxKrNQCDjnfAb6bMLFaE= github.com/Jeffail/grok v1.1.0/go.mod h1:dm0hLksrDwOMa6To7ORXCuLbuNtASIZTfYheavLpsuE= -github.com/Jeffail/shutdown v1.0.0 h1:afYjnY4pksqP/012m3NGJVccDI+WATdSzIMVHZKU8/Y= -github.com/Jeffail/shutdown v1.0.0/go.mod h1:5dT4Y1oe60SJELCkmAB1pr9uQyHBhh6cwDLQTfmuO5U= +github.com/Jeffail/shutdown v1.1.0 h1:5Bm3llKt0hnRjmTUlxgBnFg/snFfwqTOUMp3So8jCLo= +github.com/Jeffail/shutdown v1.1.0/go.mod h1:5dT4Y1oe60SJELCkmAB1pr9uQyHBhh6cwDLQTfmuO5U= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= From 7f3d8113826f14d96c5d3e68f0fa9a7011630110 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Fri, 21 Nov 2025 01:42:45 +0000 Subject: [PATCH 28/51] chore: bump changelog (#321) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b86dd5629..d33fd5c77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ Changelog All notable changes to this project will be documented in this file. +## 4.61.0 - 2025-11-21 + +### Added + +- (socket_server) Add support for unixgram (@alextreichler) + ## 4.60.0 - 2025-11-07 ### Added From eafc684fca68c638336415bf424678918c31b604 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Fri, 5 Dec 2025 09:26:50 +0100 Subject: [PATCH 29/51] pure: fix input sequence hanging when input fails --- internal/impl/pure/input_sequence.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/pure/input_sequence.go b/internal/impl/pure/input_sequence.go index 99959e643..687c67ff5 100644 --- a/internal/impl/pure/input_sequence.go +++ b/internal/impl/pure/input_sequence.go @@ -450,6 +450,7 @@ func (r *sequenceInput) createNextTarget() (input.Streamed, bool, error) { var err error r.targetMut.Lock() + defer r.targetMut.Unlock() r.target = nil if len(r.remaining) > 0 { next := r.remaining[0] @@ -466,7 +467,6 @@ func (r *sequenceInput) createNextTarget() (input.Streamed, bool, error) { r.target = target } final := len(r.remaining) == 0 - r.targetMut.Unlock() return target, final, err } From 568680af473f4d9d9aa0248b385fba88b94c1cc4 Mon Sep 17 00:00:00 2001 From: Alex Treichler Date: Fri, 5 Dec 2025 00:30:29 -0800 Subject: [PATCH 30/51] io: add new listener opts to input socket_server --- internal/impl/io/input_socket_server.go | 37 ++++++++++++++++++------- public/utils/netutil/listen.go | 3 +- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/internal/impl/io/input_socket_server.go b/internal/impl/io/input_socket_server.go index b632c3426..78ffc6470 100644 --- a/internal/impl/io/input_socket_server.go +++ b/internal/impl/io/input_socket_server.go @@ -1,5 +1,6 @@ -// Copyright 2025 Redpanda Data, Inc. +//go:build !wasm +// Copyright 2025 Redpanda Data, Inc. package io import ( @@ -24,6 +25,7 @@ import ( "github.com/redpanda-data/benthos/v4/internal/component" "github.com/redpanda-data/benthos/v4/public/service" "github.com/redpanda-data/benthos/v4/public/service/codec" + "github.com/redpanda-data/benthos/v4/public/utils/netutil" ) const ( @@ -50,13 +52,14 @@ func socketServerInputSpec() *service.ConfigSpec { Fields( service.NewStringEnumField(issFieldNetwork, "unix", "tcp", "udp", "tls", "unixgram"). Description("A network type to accept."), - service.NewStringField(isFieldAddress). + service.NewStringField(issFieldAddress). Description("The address to listen from."). Examples("/tmp/benthos.sock", "0.0.0.0:6000"), service.NewStringField(issFieldAddressCache). Description("An optional xref:components:caches/about.adoc[`cache`] within which this input should write it's bound address once known. The key of the cache item containing the address will be the label of the component suffixed with `_address` (e.g. `foo_address`), or `socket_server_address` when a label has not been provided. This is useful in situations where the address is dynamically allocated by the server (`127.0.0.1:0`) and you want to store the allocated address somewhere for reference by other systems and components."). Optional(). Version("4.25.0"), + netutil.ListenerConfigSpec(), service.NewObjectField(issFieldTLS, service.NewStringField(issFieldTLSCertFile). Description("PEM encoded certificate for use with TLS."). @@ -101,7 +104,7 @@ type wrapPacketConn struct { func (w *wrapPacketConn) Read(p []byte) (n int, err error) { n, _, err = w.ReadFrom(p) - return + return n, err } type socketServerInput struct { @@ -116,6 +119,7 @@ type socketServerInput struct { tlsSelfSigned bool tlsClientAuth tls.ClientAuthType codecCtor codec.DeprecatedFallbackCodec + listenerConf netutil.ListenerConfig messages chan service.MessageBatch shutSig *shutdown.Signaller @@ -130,10 +134,10 @@ func newSocketServerInputFromParsed(conf *service.ParsedConfig, mgr *service.Res } if t.network, err = conf.FieldString(issFieldNetwork); err != nil { - return + return i, err } if t.address, err = conf.FieldString(issFieldAddress); err != nil { - return + return i, err } t.addressCache, _ = conf.FieldString(issFieldAddressCache) @@ -155,12 +159,16 @@ func newSocketServerInputFromParsed(conf *service.ParsedConfig, mgr *service.Res case issFieldTLSClientAuthVerifyIfGiven: t.tlsClientAuth = tls.VerifyClientCertIfGiven default: - return + return i, err } if t.codecCtor, err = codec.DeprecatedCodecFromParsed(conf); err != nil { - return + return i, err } + if t.listenerConf, err = netutil.ListenerConfigFromParsed(conf.Namespace("tcp")); err != nil { + return i, err + } + return &t, nil } @@ -169,9 +177,13 @@ func (t *socketServerInput) Connect(ctx context.Context) error { var cn net.PacketConn var err error + lc := net.ListenConfig{} + if err = netutil.DecorateListenerConfig(&lc, t.listenerConf); err != nil { + return fmt.Errorf("failed to apply listener config: %w", err) + } switch t.network { case "tcp", "unix": - ln, err = net.Listen(t.network, t.address) + ln, err = lc.Listen(ctx, t.network, t.address) case "tls": var cert tls.Certificate if cert, err = loadOrCreateCertificate(t.tlsCert, t.tlsKey, t.tlsSelfSigned); err != nil { @@ -181,9 +193,14 @@ func (t *socketServerInput) Connect(ctx context.Context) error { Certificates: []tls.Certificate{cert}, ClientAuth: t.tlsClientAuth, } - ln, err = tls.Listen("tcp", t.address, config) + var baseListener net.Listener + baseListener, err = lc.Listen(ctx, "tcp", t.address) + if err != nil { + break + } + ln = tls.NewListener(baseListener, config) case "udp", "unixgram": - cn, err = net.ListenPacket(t.network, t.address) + cn, err = lc.ListenPacket(ctx, t.network, t.address) default: return fmt.Errorf("socket network '%v' is not supported by this input", t.network) } diff --git a/public/utils/netutil/listen.go b/public/utils/netutil/listen.go index e1b76de8c..b98fa2464 100644 --- a/public/utils/netutil/listen.go +++ b/public/utils/netutil/listen.go @@ -1,5 +1,6 @@ -// Copyright 2025 Redpanda Data, Inc. +//go:build !wasm +// Copyright 2025 Redpanda Data, Inc. package netutil import ( From 66df7fc642d5c766aa4925f3cba1de5c4ad4422e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 12:34:09 +0000 Subject: [PATCH 31/51] Bump actions/setup-go from 5 to 6 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5 to 6. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-go dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b9f39e532..8d243bf63 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v4 - name: Install Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: stable @@ -47,7 +47,7 @@ jobs: uses: actions/checkout@v4 - name: Install Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: stable From 2637d1576836de262122cda39f3c4c8ca04b1a92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Nov 2025 14:18:25 +0000 Subject: [PATCH 32/51] build(deps): bump golangci/golangci-lint-action from 8 to 9 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 8 to 9. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v8...v9) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-version: '9' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d243bf63..4d25282ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,7 +56,7 @@ jobs: cat .versions >> $GITHUB_ENV - name: Lint - uses: golangci/golangci-lint-action@v8 + uses: golangci/golangci-lint-action@v9 with: version: "v${{env.GOLANGCI_LINT_VERSION}}" args: "cmd/... internal/... public/..." From ef116b5ae518b1c793f8621947d6e8d83be43e37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 08:33:38 +0000 Subject: [PATCH 33/51] build(deps): bump actions/checkout from 4 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/govulncheck.yml | 2 +- .github/workflows/test.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml index db326a5b4..35af59348 100644 --- a/.github/workflows/govulncheck.yml +++ b/.github/workflows/govulncheck.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Run Go Vulnerability Check uses: golang/govulncheck-action@v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4d25282ed..c4b8d5751 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Install Go uses: actions/setup-go@v6 @@ -44,7 +44,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Install Go uses: actions/setup-go@v6 From 6f297eb58034429689fa20ae4300b65858ebb66b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Tue, 2 Dec 2025 15:35:02 +0100 Subject: [PATCH 34/51] cli: add support for listing bloblang functions and methods with jsonschema $ ./target/benthos list --format jsonschema bloblang-functions uuid_v4 now | jq { "bloblang-functions": [ { "status": "stable", "category": "Environment", "name": "now", "description": "Returns the current timestamp as a string in RFC 3339 format with the local timezone. Use the method `ts_format` in order to change the format and timezone.", "params": {}, "examples": [ { "mapping": "root.received_at = now()", "summary": "", "results": [], "skip_testing": false }, { "mapping": "root.received_at = now().ts_format(\"Mon Jan 2 15:04:05 -0700 MST 2006\", \"UTC\")", "summary": "", "results": [], "skip_testing": false } ], "impure": false }, { "status": "stable", "category": "General", "name": "uuid_v4", "description": "Generates a new RFC-4122 UUID each time it is invoked and prints a string representation.", "params": {}, "examples": [ { "mapping": "root.id = uuid_v4()", "summary": "", "results": [], "skip_testing": false } ], "impure": false } ] } --- internal/cli/list.go | 57 +++++++- internal/cli/list_test.go | 265 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 321 insertions(+), 1 deletion(-) create mode 100644 internal/cli/list_test.go diff --git a/internal/cli/list.go b/internal/cli/list.go index e41a8e0ca..4d04d5355 100644 --- a/internal/cli/list.go +++ b/internal/cli/list.go @@ -11,6 +11,7 @@ import ( "golang.org/x/text/cases" "golang.org/x/text/language" + "github.com/redpanda-data/benthos/v4/internal/bloblang/query" "github.com/redpanda-data/benthos/v4/internal/cli/common" "github.com/redpanda-data/benthos/v4/internal/config/schema" "github.com/redpanda-data/benthos/v4/internal/cuegen" @@ -48,7 +49,16 @@ components will be shown. {{.BinaryName}} list {{.BinaryName}} list --format json inputs output - {{.BinaryName}} list rate-limits buffers`)[1:], + {{.BinaryName}} list rate-limits buffers + +When using --format jsonschema with bloblang-functions or bloblang-methods, +you can optionally specify function or method names to retrieve metadata for +only those specific items: + + {{.BinaryName}} list --format jsonschema bloblang-functions + {{.BinaryName}} list --format jsonschema bloblang-functions uuid_v4 + {{.BinaryName}} list --format jsonschema bloblang-functions uuid_v4 nanoid + {{.BinaryName}} list --format jsonschema bloblang-methods uppercase catch`)[1:], Before: func(c *cli.Context) error { return common.PreApplyEnvFilesAndTemplates(c, opts) }, @@ -129,6 +139,43 @@ func listComponents(c *cli.Context, opts *common.CLIOpts) { } fmt.Fprintln(opts.Stdout, string(jsonBytes)) case "jsonschema": + // Special handling for bloblang functions and methods + if _, isFunctions := ofTypes["bloblang-functions"]; isFunctions { + functions := schema.BloblangFunctions + + // Filter by specific function names if provided + if len(ofTypes) > 1 { + filtered := []query.FunctionSpec{} + for _, fn := range functions { + if _, requested := ofTypes[fn.Name]; requested { + filtered = append(filtered, fn) + } + } + functions = filtered + } + + outputBloblangJSON(opts, "bloblang-functions", functions) + return + } + if _, isMethods := ofTypes["bloblang-methods"]; isMethods { + methods := schema.BloblangMethods + + // Filter by specific method names if provided + if len(ofTypes) > 1 { + filtered := []query.MethodSpec{} + for _, method := range methods { + if _, requested := ofTypes[method.Name]; requested { + filtered = append(filtered, method) + } + } + methods = filtered + } + + outputBloblangJSON(opts, "bloblang-methods", methods) + return + } + + // Default: Component jsonschema jsonSchemaBytes, err := jsonschema.Marshal(schema.Config, opts.Environment) if err != nil { panic(err) @@ -142,3 +189,11 @@ func listComponents(c *cli.Context, opts *common.CLIOpts) { fmt.Fprintln(opts.Stdout, string(source)) } } + +func outputBloblangJSON(opts *common.CLIOpts, key string, data any) { + jsonBytes, err := json.Marshal(map[string]any{key: data}) + if err != nil { + panic(err) + } + fmt.Fprintln(opts.Stdout, string(jsonBytes)) +} diff --git a/internal/cli/list_test.go b/internal/cli/list_test.go new file mode 100644 index 000000000..f3c8d14ec --- /dev/null +++ b/internal/cli/list_test.go @@ -0,0 +1,265 @@ +// Copyright 2025 Redpanda Data, Inc. + +package cli_test + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + icli "github.com/redpanda-data/benthos/v4/internal/cli" + "github.com/redpanda-data/benthos/v4/internal/cli/common" + + _ "github.com/redpanda-data/benthos/v4/public/components/io" + _ "github.com/redpanda-data/benthos/v4/public/components/pure" +) + +func executeListSubcmd(args []string) (string, error) { + var buf bytes.Buffer + + opts := common.NewCLIOpts("1.2.3", "now") + opts.Stdout = &buf + opts.Stderr = &buf + + err := icli.App(opts).Run(args) + return buf.String(), err +} + +func TestListBloblangFunctionsJSONSchema(t *testing.T) { + outStr, err := executeListSubcmd([]string{"benthos", "list", "--format", "jsonschema", "bloblang-functions"}) + require.NoError(t, err) + + // Parse JSON output + var result map[string][]map[string]any + err = json.Unmarshal([]byte(outStr), &result) + require.NoError(t, err, "Output should be valid JSON") + + // Verify structure + functions, ok := result["bloblang-functions"] + require.True(t, ok, "Output should contain 'bloblang-functions' key") + require.NotEmpty(t, functions, "Should have at least one function") + + // Verify first function has expected fields + firstFunc := functions[0] + assert.Contains(t, firstFunc, "name", "Function should have 'name' field") + assert.Contains(t, firstFunc, "description", "Function should have 'description' field") + assert.Contains(t, firstFunc, "status", "Function should have 'status' field") + assert.Contains(t, firstFunc, "category", "Function should have 'category' field") + assert.Contains(t, firstFunc, "params", "Function should have 'params' field") + assert.Contains(t, firstFunc, "examples", "Function should have 'examples' field") + assert.Contains(t, firstFunc, "impure", "Function should have 'impure' field") + + // Find and verify specific function (uuid_v4) + var uuidFunc map[string]any + for _, fn := range functions { + if fn["name"] == "uuid_v4" { + uuidFunc = fn + break + } + } + require.NotNil(t, uuidFunc, "Should find uuid_v4 function") + assert.Equal(t, "stable", uuidFunc["status"]) + assert.Equal(t, "General", uuidFunc["category"]) + assert.Contains(t, uuidFunc["description"], "UUID") + assert.False(t, uuidFunc["impure"].(bool)) +} + +func TestListBloblangMethodsJSONSchema(t *testing.T) { + outStr, err := executeListSubcmd([]string{"benthos", "list", "--format", "jsonschema", "bloblang-methods"}) + require.NoError(t, err) + + // Parse JSON output + var result map[string][]map[string]any + err = json.Unmarshal([]byte(outStr), &result) + require.NoError(t, err, "Output should be valid JSON") + + // Verify structure + methods, ok := result["bloblang-methods"] + require.True(t, ok, "Output should contain 'bloblang-methods' key") + require.NotEmpty(t, methods, "Should have at least one method") + + // Verify first method has expected fields + firstMethod := methods[0] + assert.Contains(t, firstMethod, "name", "Method should have 'name' field") + assert.Contains(t, firstMethod, "status", "Method should have 'status' field") + assert.Contains(t, firstMethod, "params", "Method should have 'params' field") + assert.Contains(t, firstMethod, "categories", "Method should have 'categories' field") + assert.Contains(t, firstMethod, "impure", "Method should have 'impure' field") + + // Find and verify specific method (uppercase) + var uppercaseMethod map[string]any + for _, method := range methods { + if method["name"] == "uppercase" { + uppercaseMethod = method + break + } + } + require.NotNil(t, uppercaseMethod, "Should find uppercase method") + assert.Equal(t, "stable", uppercaseMethod["status"]) + assert.False(t, uppercaseMethod["impure"].(bool)) + + // Verify categories is an array + categories, ok := uppercaseMethod["categories"].([]any) + require.True(t, ok, "Categories should be an array") + require.NotEmpty(t, categories, "Should have at least one category") +} + +func TestListBloblangFunctionWithParams(t *testing.T) { + outStr, err := executeListSubcmd([]string{"benthos", "list", "--format", "jsonschema", "bloblang-functions"}) + require.NoError(t, err) + + // Parse JSON output + var result map[string][]map[string]any + err = json.Unmarshal([]byte(outStr), &result) + require.NoError(t, err) + + functions := result["bloblang-functions"] + + // Find nanoid function which has optional parameters + var nanoidFunc map[string]any + for _, fn := range functions { + if fn["name"] == "nanoid" { + nanoidFunc = fn + break + } + } + require.NotNil(t, nanoidFunc, "Should find nanoid function") + + // Verify params structure + params, ok := nanoidFunc["params"].(map[string]any) + require.True(t, ok, "Params should be a map") + + named, ok := params["named"].([]any) + require.True(t, ok, "Named params should be an array") + require.NotEmpty(t, named, "Should have named parameters") + + // Verify first parameter has required fields + firstParam := named[0].(map[string]any) + assert.Contains(t, firstParam, "name", "Parameter should have 'name' field") + assert.Contains(t, firstParam, "description", "Parameter should have 'description' field") + assert.Contains(t, firstParam, "type", "Parameter should have 'type' field") + assert.Contains(t, firstParam, "is_optional", "Parameter should have 'is_optional' field") +} + +func TestListBloblangMethodWithParams(t *testing.T) { + outStr, err := executeListSubcmd([]string{"benthos", "list", "--format", "jsonschema", "bloblang-methods"}) + require.NoError(t, err) + + // Parse JSON output + var result map[string][]map[string]any + err = json.Unmarshal([]byte(outStr), &result) + require.NoError(t, err) + + methods := result["bloblang-methods"] + + // Find catch method which has parameters + var catchMethod map[string]any + for _, method := range methods { + if method["name"] == "catch" { + catchMethod = method + break + } + } + require.NotNil(t, catchMethod, "Should find catch method") + + // Verify params structure + params, ok := catchMethod["params"].(map[string]any) + require.True(t, ok, "Params should be a map") + + named, ok := params["named"].([]any) + require.True(t, ok, "Named params should be an array") + require.NotEmpty(t, named, "Should have named parameters") + + // Verify parameter structure + firstParam := named[0].(map[string]any) + assert.Equal(t, "fallback", firstParam["name"]) + assert.Equal(t, "query expression", firstParam["type"]) + assert.Contains(t, firstParam["description"], "query") + + // Verify examples + examples, ok := catchMethod["examples"].([]any) + require.True(t, ok, "Examples should be an array") + require.NotEmpty(t, examples, "Should have examples") +} + +func TestListBloblangFunctionFiltering(t *testing.T) { + // Test filtering to a single function + outStr, err := executeListSubcmd([]string{"benthos", "list", "--format", "jsonschema", "bloblang-functions", "uuid_v4"}) + require.NoError(t, err) + + // Parse JSON output + var result map[string][]map[string]any + err = json.Unmarshal([]byte(outStr), &result) + require.NoError(t, err) + + functions := result["bloblang-functions"] + require.Len(t, functions, 1, "Should return exactly one function") + assert.Equal(t, "uuid_v4", functions[0]["name"]) + + // Test filtering to multiple functions + outStr, err = executeListSubcmd([]string{"benthos", "list", "--format", "jsonschema", "bloblang-functions", "uuid_v4", "nanoid"}) + require.NoError(t, err) + + err = json.Unmarshal([]byte(outStr), &result) + require.NoError(t, err) + + functions = result["bloblang-functions"] + require.Len(t, functions, 2, "Should return exactly two functions") + + names := []string{functions[0]["name"].(string), functions[1]["name"].(string)} + assert.Contains(t, names, "uuid_v4") + assert.Contains(t, names, "nanoid") +} + +func TestListBloblangMethodFiltering(t *testing.T) { + // Test filtering to a single method + outStr, err := executeListSubcmd([]string{"benthos", "list", "--format", "jsonschema", "bloblang-methods", "uppercase"}) + require.NoError(t, err) + + // Parse JSON output + var result map[string][]map[string]any + err = json.Unmarshal([]byte(outStr), &result) + require.NoError(t, err) + + methods := result["bloblang-methods"] + require.Len(t, methods, 1, "Should return exactly one method") + assert.Equal(t, "uppercase", methods[0]["name"]) + + // Test filtering to multiple methods + outStr, err = executeListSubcmd([]string{"benthos", "list", "--format", "jsonschema", "bloblang-methods", "uppercase", "catch"}) + require.NoError(t, err) + + err = json.Unmarshal([]byte(outStr), &result) + require.NoError(t, err) + + methods = result["bloblang-methods"] + require.Len(t, methods, 2, "Should return exactly two methods") + + names := []string{methods[0]["name"].(string), methods[1]["name"].(string)} + assert.Contains(t, names, "uppercase") + assert.Contains(t, names, "catch") +} + +func TestListJSONSchemaComponentsNotAffected(t *testing.T) { + // Verify that regular component jsonschema output still works + outStr, err := executeListSubcmd([]string{"benthos", "list", "--format", "jsonschema"}) + require.NoError(t, err) + + // Should be a JSON schema document (not our bloblang metadata format) + var result map[string]any + err = json.Unmarshal([]byte(outStr), &result) + require.NoError(t, err, "Output should be valid JSON") + + // Should NOT have bloblang-functions or bloblang-methods keys + _, hasFunctions := result["bloblang-functions"] + _, hasMethods := result["bloblang-methods"] + assert.False(t, hasFunctions, "Component schema should not have bloblang-functions") + assert.False(t, hasMethods, "Component schema should not have bloblang-methods") + + // Should have typical JSON schema fields (definitions and properties) + assert.Contains(t, result, "definitions", "Should be a JSON schema document with definitions") + assert.Contains(t, result, "properties", "Should be a JSON schema document with properties") +} From bd675c03793400986c29897def97ae2dfd891992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Wed, 3 Dec 2025 16:25:09 +0100 Subject: [PATCH 35/51] cli/blobl: add input field $ rpk connect blobl -h NAME: redpanda-connect blobl - Execute a Redpanda Connect mapping on documents consumed via stdin or file USAGE: redpanda-connect blobl [command options] DESCRIPTION: Provides a convenient tool for mapping JSON documents over the command line: cat documents.jsonl | redpanda-connect blobl 'foo.bar.map_each(this.uppercase())' echo '{"foo":"bar"}' | redpanda-connect blobl -f ./mapping.blobl redpanda-connect blobl -i input.jsonl -f ./mapping.blobl Find out more about Bloblang at: https://docs.redpanda.com/redpanda-connect/guides/bloblang/about COMMANDS: server EXPERIMENTAL: Run a web server that hosts a Bloblang app help, h Shows a list of commands or help for one command OPTIONS: --threads value, -t value the number of processing threads to use, when >1 ordering is no longer guaranteed. (default: 1) --raw, -r consume raw strings. (default: false) --pretty, -p pretty-print output. (default: false) --file value, -f value execute a mapping from a file. --input value, -i value read input from a file instead of stdin. --output value, -o value write output to a file instead of stdout. --max-token-length value Set the buffer size for document lines. (default: 65536) --env-file value, -e value [ --env-file value, -e value ] import environment variables from a dotenv file --help, -h --- internal/cli/blobl/cli.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/cli/blobl/cli.go b/internal/cli/blobl/cli.go index 1ab6eead4..d16c5b394 100644 --- a/internal/cli/blobl/cli.go +++ b/internal/cli/blobl/cli.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "io" "os" "github.com/Jeffail/gabs/v2" @@ -31,7 +32,7 @@ var red = color.New(color.FgRed).SprintFunc() func CliCommand(opts *common.CLIOpts) *cli.Command { return &cli.Command{ Name: "blobl", - Usage: opts.ExecTemplate("Execute a {{.ProductName}} mapping on documents consumed via stdin"), + Usage: opts.ExecTemplate("Execute a {{.ProductName}} mapping on documents consumed via stdin or file"), Description: opts.ExecTemplate(` Provides a convenient tool for mapping JSON documents over the command line: @@ -39,6 +40,8 @@ Provides a convenient tool for mapping JSON documents over the command line: echo '{"foo":"bar"}' | {{.BinaryName}} blobl -f ./mapping.blobl + {{.BinaryName}} blobl -i input.jsonl -f ./mapping.blobl + Find out more about Bloblang at: {{.DocumentationURL}}/guides/bloblang/about`)[1:], Flags: []cli.Flag{ &cli.IntFlag{ @@ -62,6 +65,11 @@ Find out more about Bloblang at: {{.DocumentationURL}}/guides/bloblang/about`)[1 Aliases: []string{"f"}, Usage: "execute a mapping from a file.", }, + &cli.StringFlag{ + Name: "input", + Aliases: []string{"i"}, + Usage: "read input from a file instead of stdin.", + }, &cli.IntFlag{ Name: "max-token-length", Usage: "Set the buffer size for document lines.", @@ -280,7 +288,16 @@ func run(c *cli.Context, opts *common.CLIOpts) error { eGroup.Go(func() error { defer close(inputsChan) - scanner := bufio.NewScanner(os.Stdin) + var r io.Reader = os.Stdin + if filePath := c.String("input"); filePath != "" { + f, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open input file: %w", err) + } + defer f.Close() + r = f + } + scanner := bufio.NewScanner(r) scanner.Buffer(nil, c.Int("max-token-length")) for scanner.Scan() { input := make([]byte, len(scanner.Bytes())) From 53660b2dd3615b216621720b3388a9516423a983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Wed, 3 Dec 2025 17:16:51 +0100 Subject: [PATCH 36/51] cli/blobl: fix data race where program exits before printing output --- internal/cli/blobl/cli.go | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/internal/cli/blobl/cli.go b/internal/cli/blobl/cli.go index d16c5b394..5109c9dba 100644 --- a/internal/cli/blobl/cli.go +++ b/internal/cli/blobl/cli.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "os" + "sync" "github.com/Jeffail/gabs/v2" "github.com/fatih/color" @@ -282,10 +283,9 @@ func run(c *cli.Context, opts *common.CLIOpts) error { return errors.New(err.Error()) } - eGroup, _ := errgroup.WithContext(c.Context) - inputsChan := make(chan []byte) - eGroup.Go(func() error { + eg, _ := errgroup.WithContext(c.Context) + eg.Go(func() error { defer close(inputsChan) var r io.Reader = os.Stdin @@ -307,33 +307,28 @@ func run(c *cli.Context, opts *common.CLIOpts) error { return scanner.Err() }) - resultsChan := make(chan string) - go func() { - for res := range resultsChan { - fmt.Fprintln(opts.Stdout, res) - } - }() + var mu sync.Mutex for i := 0; i < t; i++ { - eGroup.Go(func() error { + eg.Go(func() error { execCache := newExecCache() for { input, open := <-inputsChan if !open { return nil } + res, err := execCache.executeMapping(exec, raw, pretty, input) - resultStr, err := execCache.executeMapping(exec, raw, pretty, input) + mu.Lock() if err != nil { fmt.Fprintln(opts.Stderr, red(fmt.Sprintf("failed to execute map: %v", err))) - continue + } else { + fmt.Fprintln(opts.Stdout, res) } - resultsChan <- resultStr + mu.Unlock() } }) } - err = eGroup.Wait() - close(resultsChan) - return err + return eg.Wait() } From edae09d0ee75b605e4fc7aa657050f8bba5a26ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Thu, 4 Dec 2025 10:56:23 +0100 Subject: [PATCH 37/51] bloblang: function and method descriptions and examples overhaul Core Benthos Functions/Methods (94 items) 1. Functions (25): batch_index, batch_size, bytes, content, counter, deleted, env, error, errored, error_source_*, file, file_rel, hostname, json, ksuid, metadata, nanoid, nothing, now, pi, random_int, range, throw, timestamp_unix*, tracing_id, tracing_span, uuid_v4, uuid_v7, var 2. String Methods (31): capitalize, contains, escape_html, escape_url_query, filepath_join, filepath_split, format, has_prefix, has_suffix, index_of, join, lowercase, quote, unquote, repeat, replace, replace_all, replace_many, replace_all_many, reverse, re_find_all, re_match, re_replace*, split, string, trim, trim_prefix, trim_suffix, unescape_html, unescape_url_query, uppercase 3. Structured Methods (28): all, any, append, assign, collapse, enumerated, exists, explode, filter, find, find_all, find_by, find_all_by, flatten, fold, key_values, keys, length, map_each, map_each_key, merge, sort, sort_by, sum, unique, values, without 4. Number Methods (10): ceil, floor, log, log10, max, min, round, bitwise_and, bitwise_or, bitwise_xor Benthos Implementation Methods (22 items) 1. Encoding Methods (2): compress, decompress 2. Time Methods (15): ts_round, ts_tz, ts_add_iso8601, ts_sub_iso8601, parse_duration, parse_duration_iso8601, ts_parse, ts_strptime, ts_format, ts_strftime, ts_unix, ts_unix_milli, ts_unix_micro, ts_unix_nano, ts_sub 3. General Functions (1): counter 4. I/O Functions (4): hostname, env, file, file_rel --- internal/bloblang/query/functions.go | 178 +++++++++++--- internal/bloblang/query/methods_numbers.go | 90 ++++--- internal/bloblang/query/methods_strings.go | 142 +++++++++--- internal/bloblang/query/methods_structured.go | 169 ++++++++++---- internal/impl/io/bloblang.go | 58 +++-- internal/impl/pure/bloblang_encoding.go | 39 ++-- internal/impl/pure/bloblang_general.go | 65 +++--- internal/impl/pure/bloblang_time.go | 219 ++++++++++-------- 8 files changed, 645 insertions(+), 315 deletions(-) diff --git a/internal/bloblang/query/functions.go b/internal/bloblang/query/functions.go index ac883df9f..13e63b70a 100644 --- a/internal/bloblang/query/functions.go +++ b/internal/bloblang/query/functions.go @@ -186,10 +186,13 @@ func NewLiteralFunction(annotation string, v any) *Literal { var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryMessage, "batch_index", - "Returns the index of the mapped message within a batch. This is useful for applying maps only on certain messages of a batch.", + "Returns the zero-based index of the current message within its batch. Use this to conditionally process messages based on their position, or to create sequential identifiers within a batch.", NewExampleSpec("", `root = if batch_index() > 0 { deleted() }`, ), + NewExampleSpec("Create a unique identifier combining batch position with timestamp.", + `root.id = "%v-%v".format(timestamp_unix(), batch_index())`, + ), ), func(ctx FunctionContext) (any, error) { return int64(ctx.Index), nil @@ -201,9 +204,12 @@ var _ = registerSimpleFunction( var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryMessage, "batch_size", - "Returns the size of the message batch.", + "Returns the total number of messages in the current batch. Use this to determine batch boundaries or compute relative positions.", NewExampleSpec("", - `root.foo = batch_size()`, + `root.total = batch_size()`, + ), + NewExampleSpec("Check if processing the last message in a batch.", + `root.is_last = batch_index() == batch_size() - 1`, ), ), func(ctx FunctionContext) (any, error) { @@ -216,12 +222,18 @@ var _ = registerSimpleFunction( var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryMessage, "content", - "Returns the full raw contents of the mapping target message as a byte array. When mapping to a JSON field the value should be encoded using the method xref:guides:bloblang/methods.adoc#encode[`encode`], or cast to a string directly using the method xref:guides:bloblang/methods.adoc#string[`string`], otherwise it will be base64 encoded by default.", + "Returns the raw message payload as bytes, regardless of the current mapping context. Use this to access the original message when working within nested contexts, or to store the entire message as a field.", NewExampleSpec("", `root.doc = content().string()`, `{"foo":"bar"}`, `{"doc":"{\"foo\":\"bar\"}"}`, ), + NewExampleSpec("Preserve original message while adding metadata.", + `root.original = content().string() +root.processed_by = "ai"`, + `{"foo":"bar"}`, + `{"original":"{\"foo\":\"bar\"}","processed_by":"ai"}`, + ), ), func(ctx FunctionContext) (any, error) { return ctx.MsgBatch.Get(ctx.Index).AsBytes(), nil @@ -233,12 +245,15 @@ var _ = registerSimpleFunction( var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryMessage, "tracing_span", - "Provides the message tracing span xref:components:tracers/about.adoc[(created via Open Telemetry APIs)] as an object serialized via text map formatting. The returned value will be `null` if the message does not have a span.", + "Returns the OpenTelemetry tracing span attached to the message as a text map object, or `null` if no span exists. Use this to propagate trace context to downstream systems via headers or metadata.", NewExampleSpec("", `root.headers.traceparent = tracing_span().traceparent`, `{"some_stuff":"just can't be explained by science"}`, `{"headers":{"traceparent":"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}}`, ), + NewExampleSpec("Forward all tracing fields to output metadata.", + `meta = tracing_span()`, + ), ).Experimental(), func(fCtx FunctionContext) (any, error) { span := tracing.GetSpan(fCtx.MsgBatch.Get(fCtx.Index)) @@ -254,10 +269,14 @@ var _ = registerSimpleFunction( var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryMessage, "tracing_id", - "Provides the message trace id. The returned value will be zeroed if the message does not contain a span.", + "Returns the OpenTelemetry trace ID for the message, or an empty string if no tracing span exists. Use this to correlate logs and events with distributed traces.", NewExampleSpec("", `meta trace_id = tracing_id()`, ), + NewExampleSpec("Add trace ID to structured logs.", + `root.log_entry = this +root.log_entry.trace_id = tracing_id()`, + ), ).Experimental(), func(fCtx FunctionContext) (any, error) { traceID := tracing.GetTraceID(fCtx.MsgBatch.Get(fCtx.Index)) @@ -316,7 +335,7 @@ func countFunction(args *ParsedParams) (Function, error) { var _ = registerFunction( NewFunctionSpec( FunctionCategoryGeneral, "deleted", - "A function that returns a result indicating that the mapping target should be deleted. Deleting, also known as dropping, messages will result in them being acknowledged as successfully processed to inputs in a Redpanda Connect pipeline. For more information about error handling patterns read xref:configuration:error_handling.adoc[].", + "Returns a deletion marker that removes the target field or message. When applied to root, the entire message is dropped while still being acknowledged as successfully processed. Use this to filter data or conditionally remove fields.", NewExampleSpec("", `root = this root.bar = deleted()`, @@ -324,7 +343,7 @@ root.bar = deleted()`, `{"baz":"baz_value","foo":"foo value"}`, ), NewExampleSpec( - "Since the result is a value it can be used to do things like remove elements of an array within `map_each`.", + "Filter array elements by returning deleted for unwanted items.", `root.new_nums = this.nums.map_each(num -> if num < 10 { deleted() } else { num - 10 })`, `{"nums":[3,11,4,17]}`, `{"new_nums":[1,7]}`, @@ -340,10 +359,15 @@ root.bar = deleted()`, var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryMessage, "error", - "If an error has occurred during the processing of a message this function returns the reported cause of the error as a string, otherwise `null`. For more information about error handling patterns read xref:configuration:error_handling.adoc[].", + "Returns the error message string if the message has failed processing, otherwise `null`. Use this in error handling pipelines to log or route failed messages based on their error details.", NewExampleSpec("", `root.doc.error = error()`, ), + NewExampleSpec("Route messages to different outputs based on error presence.", + `root = this +root.error_msg = error() +root.has_error = error() != null`, + ), ), func(ctx FunctionContext) (any, error) { if err := ctx.MsgBatch.Get(ctx.Index).ErrorGet(); err != nil { @@ -356,10 +380,13 @@ var _ = registerSimpleFunction( var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryMessage, "errored", - "Returns a boolean value indicating whether an error has occurred during the processing of a message. For more information about error handling patterns read xref:configuration:error_handling.adoc[].", + "Returns true if the message has failed processing, false otherwise. Use this for conditional logic in error handling workflows or to route failed messages to dead letter queues.", NewExampleSpec("", `root.doc.status = if errored() { 400 } else { 200 }`, ), + NewExampleSpec("Send only failed messages to a separate stream.", + `root = if errored() { this } else { deleted() }`, + ), ), func(ctx FunctionContext) (any, error) { return ctx.MsgBatch.Get(ctx.Index).ErrorGet() != nil, nil @@ -369,10 +396,19 @@ var _ = registerSimpleFunction( var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryMessage, "error_source_name", - "Returns the name of the source component which raised the error during the processing of a message. `null` is returned when the error is null or no source component is associated with it. For more information about error handling patterns read xref:configuration:error_handling.adoc[].", + "Returns the component name that caused the error, or `null` if the message has no error or the error has no associated component. Use this to identify which processor or component in your pipeline caused a failure.", NewExampleSpec("", `root.doc.error_source_name = error_source_name()`, ), + NewExampleSpec("Create detailed error logs with component information.", + `root.error_details = if errored() { + { + "message": error(), + "component": error_source_name(), + "timestamp": now() + } +}`, + ), ), func(ctx FunctionContext) (any, error) { if err := ctx.MsgBatch.Get(ctx.Index).ErrorGet(); err != nil { @@ -387,10 +423,13 @@ var _ = registerSimpleFunction( var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryMessage, "error_source_label", - "Returns the label of the source component which raised the error during the processing of a message or an empty string if not set. `null` is returned when the error is null or no source component is associated with it. For more information about error handling patterns read xref:configuration:error_handling.adoc[].", + "Returns the user-defined label of the component that caused the error, empty string if no label is set, or `null` if the message has no error. Use this for more human-readable error tracking when components have custom labels.", NewExampleSpec("", `root.doc.error_source_label = error_source_label()`, ), + NewExampleSpec("Route errors based on component labels.", + `root.error_category = error_source_label().or("unknown")`, + ), ), func(ctx FunctionContext) (any, error) { if err := ctx.MsgBatch.Get(ctx.Index).ErrorGet(); err != nil { @@ -405,10 +444,17 @@ var _ = registerSimpleFunction( var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryMessage, "error_source_path", - "Returns the path of the source component which raised the error during the processing of a message. `null` is returned when the error is null or no source component is associated with it. For more information about error handling patterns read xref:configuration:error_handling.adoc[].", + "Returns the dot-separated path to the component that caused the error, or `null` if the message has no error. Use this to identify the exact location of a failed component in nested pipeline configurations.", NewExampleSpec("", `root.doc.error_source_path = error_source_path()`, ), + NewExampleSpec("Build comprehensive error context for debugging.", + `root.error_info = { + "path": error_source_path(), + "component": error_source_name(), + "message": error() +}`, + ), ), func(ctx FunctionContext) (any, error) { if err := ctx.MsgBatch.Get(ctx.Index).ErrorGet(); err != nil { @@ -425,7 +471,7 @@ var _ = registerSimpleFunction( var _ = registerFunction( NewFunctionSpec( FunctionCategoryGeneral, "range", - "The `range` function creates an array of integers following a range between a start, stop and optional step integer argument. If the step argument is omitted then it defaults to 1. A negative step can be provided as long as stop < start.", + "Creates an array of integers from start (inclusive) to stop (exclusive) with an optional step. Use this to generate sequences for iteration, indexing, or creating numbered lists.", NewExampleSpec("", `root.a = range(0, 10) root.b = range(start: 0, stop: this.max, step: 2) # Using named params @@ -433,6 +479,14 @@ root.c = range(0, -this.max, -2)`, `{"max":10}`, `{"a":[0,1,2,3,4,5,6,7,8,9],"b":[0,2,4,6,8],"c":[0,-2,-4,-6,-8]}`, ), + NewExampleSpec("Generate a sequence for batch processing.", + `root.pages = range(0, this.total_items, 100).map_each(offset -> { + "offset": offset, + "limit": 100 +})`, + `{"total_items":250}`, + `{"pages":[{"limit":100,"offset":0},{"limit":100,"offset":100}]}`, + ), ). Param(ParamInt64("start", "The start value.")). Param(ParamInt64("stop", "The stop value.")). @@ -475,14 +529,14 @@ func rangeFunction(args *ParsedParams) (Function, error) { var _ = registerFunction( NewFunctionSpec( FunctionCategoryMessage, "json", - "Returns the value of a field within a JSON message located by a [dot path][field_paths] argument. This function always targets the entire source JSON document regardless of the mapping context.", + "Returns a field from the original JSON message by dot path, always accessing the root document regardless of mapping context. Use this to reference the source message when working in nested contexts or to extract specific fields.", NewExampleSpec("", `root.mapped = json("foo.bar")`, `{"foo":{"bar":"hello world"}}`, `{"mapped":"hello world"}`, ), NewExampleSpec( - "The path argument is optional and if omitted the entire JSON payload is returned.", + "Access the original message from within nested mapping contexts.", `root.doc = json()`, `{"foo":{"bar":"hello world"}}`, `{"doc":{"foo":{"bar":"hello world"}}}`, @@ -563,12 +617,20 @@ func NewMetaFunction(key string) Function { var _ = registerFunction( NewFunctionSpec( FunctionCategoryMessage, "metadata", - "Returns the value of a metadata key from the input message, or `null` if the key does not exist. Since values are extracted from the read-only input message they do NOT reflect changes made from within the map, in order to query metadata mutations made within a mapping use the xref:guides:bloblang/about.adoc#metadata[`@` operator]. This function supports extracting metadata from other messages of a batch with the `from` method.", + "Returns metadata from the input message by key, or `null` if the key doesn't exist. This reads the original metadata; to access modified metadata during mapping, use the `@` operator instead. Use this to extract message properties like topics, headers, or timestamps.", NewExampleSpec("", `root.topic = metadata("kafka_topic")`), NewExampleSpec( - "The key parameter is optional and if omitted the entire metadata contents are returned as an object.", + "Retrieve all metadata as an object by omitting the key parameter.", `root.all_metadata = metadata()`, ), + NewExampleSpec( + "Copy specific metadata fields to the message body.", + `root.source = { + "topic": metadata("kafka_topic"), + "partition": metadata("kafka_partition"), + "timestamp": metadata("kafka_timestamp_unix") +}`, + ), ).Param(ParamString("key", "An optional key of a metadata value to obtain.").Default("")), func(args *ParsedParams) (Function, error) { key, err := args.FieldString("key") @@ -729,9 +791,9 @@ var _ = registerFunction( var _ = registerFunction( NewFunctionSpec( FunctionCategoryGeneral, "random_int", ` -Generates a non-negative pseudo-random 64-bit integer. An optional integer argument can be provided in order to seed the random number generator. +Generates a pseudo-random non-negative 64-bit integer. Use this for creating random IDs, sampling data, or generating test values. Provide a seed for reproducible randomness, or use a dynamic seed like `+"`timestamp_unix_nano()`"+` for unique values per mapping instance. -Optional `+"`min` and `max`"+` arguments can be provided in order to only generate numbers within a range. Neither of these parameters can be set via a dynamic expression (i.e. from values taken from mapped data). Instead, for dynamic ranges extract a min and max manually using a modulo operator (`+"`random_int() % a + b`"+`).`, +Optional `+"`min` and `max`"+` parameters constrain the output range (both inclusive). For dynamic ranges based on message data, use the modulo operator instead: `+"`random_int() % dynamic_max + dynamic_min`"+`.`, NewExampleSpec("", `root.first = random_int() root.second = random_int(1) @@ -741,9 +803,9 @@ root.fifth = random_int(timestamp_unix_nano(), 5, 20) root.sixth = random_int(seed:timestamp_unix_nano(), max:20) `, ), - NewExampleSpec("It is possible to specify a dynamic seed argument, in which case the argument will only be resolved once during the lifetime of the mapping.", - `root.first = random_int(timestamp_unix_nano())`, - `root.second = random_int(timestamp_unix_nano(), 5, 20)`, + NewExampleSpec("Use a dynamic seed for unique random values per mapping instance.", + `root.random_id = random_int(timestamp_unix_nano()) +root.sample_percent = random_int(seed: timestamp_unix_nano(), min: 0, max: 100)`, ), ). Param(ParamQuery( @@ -809,11 +871,11 @@ func randomIntFunction(args *ParsedParams) (Function, error) { var _ = registerFunction( NewFunctionSpec( FunctionCategoryEnvironment, "now", - "Returns the current timestamp as a string in RFC 3339 format with the local timezone. Use the method `ts_format` in order to change the format and timezone.", + "Returns the current timestamp as an RFC 3339 formatted string with nanosecond precision. Use this to add processing timestamps to messages or measure time between events. Chain with `ts_format` to customize the format or timezone.", NewExampleSpec("", `root.received_at = now()`, ), - NewExampleSpec("", + NewExampleSpec("Format the timestamp in a custom format and timezone.", `root.received_at = now().ts_format("Mon Jan 2 15:04:05 -0700 MST 2006", "UTC")`, ), ), @@ -827,10 +889,13 @@ var _ = registerFunction( var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryEnvironment, "timestamp_unix", - "Returns the current unix timestamp in seconds.", + "Returns the current Unix timestamp in seconds since epoch. Use this for numeric timestamps compatible with most systems, or as a seed for random number generation.", NewExampleSpec("", `root.received_at = timestamp_unix()`, ), + NewExampleSpec("Create a sortable ID combining timestamp with a counter.", + `root.id = "%v-%v".format(timestamp_unix(), batch_index())`, + ), ), func(_ FunctionContext) (any, error) { return time.Now().Unix(), nil @@ -840,10 +905,13 @@ var _ = registerSimpleFunction( var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryEnvironment, "timestamp_unix_milli", - "Returns the current unix timestamp in milliseconds.", + "Returns the current Unix timestamp in milliseconds since epoch. Use this for millisecond-precision timestamps common in web APIs and JavaScript systems.", NewExampleSpec("", `root.received_at = timestamp_unix_milli()`, ), + NewExampleSpec("Add processing time metadata.", + `meta processing_time_ms = timestamp_unix_milli()`, + ), ), func(_ FunctionContext) (any, error) { return time.Now().UnixMilli(), nil @@ -853,10 +921,13 @@ var _ = registerSimpleFunction( var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryEnvironment, "timestamp_unix_micro", - "Returns the current unix timestamp in microseconds.", + "Returns the current Unix timestamp in microseconds since epoch. Use this for high-precision timing measurements or when microsecond resolution is required.", NewExampleSpec("", `root.received_at = timestamp_unix_micro()`, ), + NewExampleSpec("Measure elapsed time between events.", + `root.processing_duration_us = timestamp_unix_micro() - this.start_time_us`, + ), ), func(_ FunctionContext) (any, error) { return time.Now().UnixMicro(), nil @@ -866,10 +937,13 @@ var _ = registerSimpleFunction( var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryEnvironment, "timestamp_unix_nano", - "Returns the current unix timestamp in nanoseconds.", + "Returns the current Unix timestamp in nanoseconds since epoch. Use this for the highest precision timing or as a unique seed value that changes on every invocation.", NewExampleSpec("", `root.received_at = timestamp_unix_nano()`, ), + NewExampleSpec("Generate unique random values on each mapping.", + `root.random_value = random_int(timestamp_unix_nano())`, + ), ), func(_ FunctionContext) (any, error) { return time.Now().UnixNano(), nil @@ -881,7 +955,7 @@ var _ = registerSimpleFunction( var _ = registerFunction( NewFunctionSpec( FunctionCategoryGeneral, "throw", - "Throws an error similar to a regular mapping error. This is useful for abandoning a mapping entirely given certain conditions.", + "Immediately fails the mapping with a custom error message. Use this to halt processing when data validation fails or required fields are missing, causing the message to be routed to error handlers.", NewExampleSpec("", `root.doc.type = match { this.exists("header.id") => "foo" @@ -894,6 +968,17 @@ root.doc.contents = (this.body.content | this.thing.body)`, `{"nothing":"matches"}`, `Error("failed assignment (line 1): unknown type")`, ), + NewExampleSpec("Validate required fields before processing.", + `root = if this.exists("user_id") { + this +} else { + throw("missing required field: user_id") +}`, + `{"user_id":123,"name":"alice"}`, + `{"name":"alice","user_id":123}`, + `{"name":"bob"}`, + `Error("failed assignment (line 1): missing required field: user_id")`, + ), ).Param(ParamString("why", "A string explanation for why an error was thrown, this will be added to the resulting error message.")), func(args *ParsedParams) (Function, error) { msg, err := args.FieldString("why") @@ -911,8 +996,12 @@ root.doc.contents = (this.body.content | this.thing.body)`, var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryGeneral, "uuid_v4", - "Generates a new RFC-4122 UUID each time it is invoked and prints a string representation.", + "Generates a random RFC-4122 version 4 UUID. Use this for creating unique identifiers that don't reveal timing information or require ordering. Each invocation produces a new globally unique ID.", NewExampleSpec("", `root.id = uuid_v4()`), + NewExampleSpec("Add unique request IDs for tracing.", + `root = this +root.request_id = uuid_v4()`, + ), ), func(_ FunctionContext) (any, error) { u4, err := uuid.NewV4() @@ -926,9 +1015,11 @@ var _ = registerSimpleFunction( var _ = registerFunction( NewFunctionSpec( FunctionCategoryGeneral, "uuid_v7", - "Generates a new time ordered UUID each time it is invoked and prints a string representation.", + "Generates a time-ordered UUID version 7 with millisecond timestamp precision. Use this for sortable unique identifiers that maintain chronological ordering, ideal for database keys or event IDs. Optionally specify a custom timestamp.", NewExampleSpec("", `root.id = uuid_v7()`), - NewExampleSpec("It is also possible to specify the timestamp for the uuid_v7", `root.id = uuid_v7(now().ts_sub_iso8601("PT1M"))`), + NewExampleSpec("Generate a UUID with a specific timestamp for backdating events.", + `root.id = uuid_v7(now().ts_sub_iso8601("PT1M"))`, + ), ).Param(ParamTimestamp("time", "An optional timestamp to use for the time ordered portion of the UUID.").Optional()), func(args *ParsedParams) (Function, error) { time, err := args.FieldOptionalTimestamp("time") @@ -957,10 +1048,10 @@ var _ = registerFunction( var _ = registerFunction( NewFunctionSpec( FunctionCategoryGeneral, "nanoid", - "Generates a new nanoid each time it is invoked and prints a string representation.", + "Generates a URL-safe unique identifier using Nano ID. Use this for compact, URL-friendly IDs with good collision resistance. Customize the length (default 21) or provide a custom alphabet for specific character requirements.", NewExampleSpec("", `root.id = nanoid()`), - NewExampleSpec("It is possible to specify an optional length parameter.", `root.id = nanoid(54)`), - NewExampleSpec("It is also possible to specify an optional custom alphabet after the length parameter.", `root.id = nanoid(54, "abcde")`), + NewExampleSpec("Generate a longer ID for additional uniqueness.", `root.id = nanoid(54)`), + NewExampleSpec("Use a custom alphabet for domain-specific IDs.", `root.id = nanoid(54, "abcde")`), ). Param(ParamInt64("length", "An optional length.").Optional()). Param(ParamString("alphabet", "An optional custom alphabet to use for generating IDs. When specified the field `length` must also be present.").Optional()), @@ -995,8 +1086,15 @@ func nanoidFunction(args *ParsedParams) (Function, error) { var _ = registerSimpleFunction( NewFunctionSpec( FunctionCategoryGeneral, "ksuid", - "Generates a new ksuid each time it is invoked and prints a string representation.", + "Generates a K-Sortable Unique Identifier with built-in timestamp ordering. Use this for distributed unique IDs that sort chronologically and remain collision-resistant without coordination between generators.", NewExampleSpec("", `root.id = ksuid()`), + NewExampleSpec("Create sortable event IDs for logging.", + `root.event = { + "id": ksuid(), + "type": this.event_type, + "data": this.payload +}`, + ), ), func(_ FunctionContext) (any, error) { return ksuid.New().String(), nil @@ -1040,11 +1138,15 @@ func NewVarFunction(name string) Function { var _ = registerFunction( NewFunctionSpec( FunctionCategoryGeneral, "bytes", - "Create a new byte array that is zero initialized", + "Creates a zero-initialized byte array of specified length. Use this to allocate fixed-size byte buffers for binary data manipulation or to generate padding.", NewExampleSpec("", `root.data = bytes(5)`, `{"data":"AAAAAAAK"}`, ), + NewExampleSpec("Create a buffer for binary operations.", + `root.header = bytes(16) +root.payload = content()`, + ), ).Param(ParamInt64("length", "The size of the resulting byte array.")), func(args *ParsedParams) (Function, error) { length, err := args.FieldInt64("length") diff --git a/internal/bloblang/query/methods_numbers.go b/internal/bloblang/query/methods_numbers.go index b93082289..87e221791 100644 --- a/internal/bloblang/query/methods_numbers.go +++ b/internal/bloblang/query/methods_numbers.go @@ -11,7 +11,7 @@ import ( ) var _ = registerSimpleMethod( - NewMethodSpec("ceil", "Returns the least integer value greater than or equal to a number. If the resulting value fits within a 64-bit integer then that is returned, otherwise a new floating point number is returned.").InCategory( + NewMethodSpec("ceil", "Rounds a number up to the nearest integer. Returns an integer if the result fits in 64-bit, otherwise returns a float.").InCategory( MethodCategoryNumbers, "", NewExampleSpec("", `root.new_value = this.value.ceil()`, @@ -20,6 +20,11 @@ var _ = registerSimpleMethod( `{"value":-5.9}`, `{"new_value":-5}`, ), + NewExampleSpec("", + `root.result = this.price.ceil()`, + `{"price":19.99}`, + `{"result":20}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return numberMethod(func(f *float64, i *int64, ui *uint64) (any, error) { @@ -40,7 +45,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( - "floor", "Returns the greatest integer value less than or equal to the target number. If the resulting value fits within a 64-bit integer then that is returned, otherwise a new floating point number is returned.", + "floor", "Rounds a number down to the nearest integer. Returns an integer if the result fits in 64-bit, otherwise returns a float.", ).InCategory( MethodCategoryNumbers, "", @@ -48,6 +53,13 @@ var _ = registerSimpleMethod( `root.new_value = this.value.floor()`, `{"value":5.7}`, `{"new_value":5}`, + `{"value":-3.2}`, + `{"new_value":-4}`, + ), + NewExampleSpec("", + `root.whole_seconds = this.duration_seconds.floor()`, + `{"duration_seconds":12.345}`, + `{"whole_seconds":12}`, ), ), func(*ParsedParams) (simpleMethod, error) { @@ -68,7 +80,7 @@ var _ = registerSimpleMethod( ) var _ = registerSimpleMethod( - NewMethodSpec("log", "Returns the natural logarithm of a number.").InCategory( + NewMethodSpec("log", "Calculates the natural logarithm (base e) of a number.").InCategory( MethodCategoryNumbers, "", NewExampleSpec("", `root.new_value = this.value.log().round()`, @@ -77,6 +89,11 @@ var _ = registerSimpleMethod( `{"value":2.7183}`, `{"new_value":1}`, ), + NewExampleSpec("", + `root.ln_result = this.number.log()`, + `{"number":10}`, + `{"ln_result":2.302585092994046}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return numberMethod(func(f *float64, i *int64, ui *uint64) (any, error) { @@ -94,7 +111,7 @@ var _ = registerSimpleMethod( ) var _ = registerSimpleMethod( - NewMethodSpec("log10", "Returns the decimal logarithm of a number.").InCategory( + NewMethodSpec("log10", "Calculates the base-10 logarithm of a number.").InCategory( MethodCategoryNumbers, "", NewExampleSpec("", `root.new_value = this.value.log10()`, @@ -103,6 +120,11 @@ var _ = registerSimpleMethod( `{"value":1000}`, `{"new_value":3}`, ), + NewExampleSpec("", + `root.log_value = this.magnitude.log10()`, + `{"magnitude":10000}`, + `{"log_value":4}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return numberMethod(func(f *float64, i *int64, ui *uint64) (any, error) { @@ -122,7 +144,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( "max", - "Returns the largest numerical value found within an array. All values must be numerical and the array must not be empty, otherwise an error is returned.", + "Returns the largest number from an array. All elements must be numbers and the array cannot be empty.", ).InCategory( MethodCategoryNumbers, "", NewExampleSpec("", @@ -131,11 +153,9 @@ var _ = registerSimpleMethod( `{"biggest":7}`, ), NewExampleSpec("", - `root.new_value = [0,this.value].max()`, - `{"value":-1}`, - `{"new_value":0}`, - `{"value":7}`, - `{"new_value":7}`, + `root.highest_temp = this.temperatures.max()`, + `{"temperatures":[20.5,22.1,19.8,23.4]}`, + `{"highest_temp":23.4}`, ), ), func(*ParsedParams) (simpleMethod, error) { @@ -165,7 +185,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( "min", - "Returns the smallest numerical value found within an array. All values must be numerical and the array must not be empty, otherwise an error is returned.", + "Returns the smallest number from an array. All elements must be numbers and the array cannot be empty.", ).InCategory( MethodCategoryNumbers, "", NewExampleSpec("", @@ -174,11 +194,9 @@ var _ = registerSimpleMethod( `{"smallest":-2.5}`, ), NewExampleSpec("", - `root.new_value = [10,this.value].min()`, - `{"value":2}`, - `{"new_value":2}`, - `{"value":23}`, - `{"new_value":10}`, + `root.lowest_temp = this.temperatures.min()`, + `{"temperatures":[20.5,22.1,19.8,23.4]}`, + `{"lowest_temp":19.8}`, ), ), func(*ParsedParams) (simpleMethod, error) { @@ -207,7 +225,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( - "round", "Rounds numbers to the nearest integer, rounding half away from zero. If the resulting value fits within a 64-bit integer then that is returned, otherwise a new floating point number is returned.", + "round", "Rounds a number to the nearest integer. Values at .5 round away from zero. Returns an integer if the result fits in 64-bit, otherwise returns a float.", ).InCategory( MethodCategoryNumbers, "", @@ -218,6 +236,11 @@ var _ = registerSimpleMethod( `{"value":5.9}`, `{"new_value":6}`, ), + NewExampleSpec("", + `root.rounded = this.score.round()`, + `{"score":87.5}`, + `{"rounded":88}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return numberMethod(func(f *float64, i *int64, ui *uint64) (any, error) { @@ -238,7 +261,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( - "bitwise_and", "Returns the number bitwise AND-ed with the specified value.", + "bitwise_and", "Performs a bitwise AND operation between the integer and the specified value.", ).InCategory( MethodCategoryNumbers, "", @@ -246,10 +269,11 @@ var _ = registerSimpleMethod( `root.new_value = this.value.bitwise_and(6)`, `{"value":12}`, `{"new_value":4}`, - `{"value":0}`, - `{"new_value":0}`, - `{"value":-4}`, - `{"new_value":4}`, + ), + NewExampleSpec("", + `root.masked = this.flags.bitwise_and(15)`, + `{"flags":127}`, + `{"masked":15}`, ), ).Param(ParamInt64("value", "The value to AND with")), func(args *ParsedParams) (simpleMethod, error) { @@ -271,7 +295,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( - "bitwise_or", "Returns the number bitwise OR-ed with the specified value.", + "bitwise_or", "Performs a bitwise OR operation between the integer and the specified value.", ).InCategory( MethodCategoryNumbers, "", @@ -279,10 +303,11 @@ var _ = registerSimpleMethod( `root.new_value = this.value.bitwise_or(6)`, `{"value":12}`, `{"new_value":14}`, - `{"value":0}`, - `{"new_value":6}`, - `{"value":-2}`, - `{"new_value":-2}`, + ), + NewExampleSpec("", + `root.combined = this.flags.bitwise_or(8)`, + `{"flags":4}`, + `{"combined":12}`, ), ).Param(ParamInt64("value", "The value to OR with")), func(args *ParsedParams) (simpleMethod, error) { @@ -304,7 +329,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( - "bitwise_xor", "Returns the number bitwise eXclusive-OR-ed with the specified value.", + "bitwise_xor", "Performs a bitwise XOR (exclusive OR) operation between the integer and the specified value.", ).InCategory( MethodCategoryNumbers, "", @@ -312,10 +337,11 @@ var _ = registerSimpleMethod( `root.new_value = this.value.bitwise_xor(6)`, `{"value":12}`, `{"new_value":10}`, - `{"value":0}`, - `{"new_value":6}`, - `{"value":-2}`, - `{"new_value":-8}`, + ), + NewExampleSpec("", + `root.toggled = this.flags.bitwise_xor(5)`, + `{"flags":3}`, + `{"toggled":6}`, ), ).Param(ParamInt64("value", "The value to XOR with")), func(args *ParsedParams) (simpleMethod, error) { diff --git a/internal/bloblang/query/methods_strings.go b/internal/bloblang/query/methods_strings.go index 650163e24..5fcdb86dc 100644 --- a/internal/bloblang/query/methods_strings.go +++ b/internal/bloblang/query/methods_strings.go @@ -67,12 +67,17 @@ var _ = registerSimpleMethod( "capitalize", "", ).InCategory( MethodCategoryStrings, - "Takes a string value and returns a copy with all Unicode letters that begin words mapped to their Unicode title case.", + "Converts the first letter of each word in a string to uppercase (title case). Useful for formatting names, titles, and headings.", NewExampleSpec("", `root.title = this.title.capitalize()`, `{"title":"the foo bar"}`, `{"title":"The Foo Bar"}`, ), + NewExampleSpec("", + `root.name = this.name.capitalize()`, + `{"name":"alice smith"}`, + `{"name":"Alice Smith"}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return func(v any, ctx FunctionContext) (any, error) { @@ -501,12 +506,17 @@ var _ = registerSimpleMethod( "escape_html", "", ).InCategory( MethodCategoryStrings, - "Escapes a string so that special characters like `<` to become `<`. It escapes only five such characters: `<`, `>`, `&`, `'` and `\"` so that it can be safely placed within an HTML entity.", + "Escapes special HTML characters (`<`, `>`, `&`, `'`, `\"`) to make a string safe for HTML output. Use when embedding untrusted text in HTML to prevent XSS vulnerabilities.", NewExampleSpec("", `root.escaped = this.value.escape_html()`, `{"value":"foo & bar"}`, `{"escaped":"foo & bar"}`, ), + NewExampleSpec("", + `root.safe_html = this.user_input.escape_html()`, + `{"user_input":""}`, + `{"safe_html":"<script>alert('xss')</script>"}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return stringMethod(func(s string) (any, error) { @@ -522,7 +532,7 @@ var _ = registerSimpleMethod( "index_of", "", ).InCategory( MethodCategoryStrings, - "Returns the starting index of the argument substring in a string target, or `-1` if the target doesn't contain the argument.", + "Finds the position of a substring within a string. Returns the zero-based index of the first occurrence, or -1 if not found. Useful for searching and string manipulation.", NewExampleSpec("", `root.index = this.thing.index_of("bar")`, `{"thing":"foobar"}`, @@ -558,12 +568,17 @@ var _ = registerSimpleMethod( "unescape_html", "", ).InCategory( MethodCategoryStrings, - "Unescapes a string so that entities like `<` become `<`. It unescapes a larger range of entities than `escape_html` escapes. For example, `á` unescapes to `á`, as does `á` and `&xE1;`.", + "Converts HTML entities back to their original characters. Handles named entities (`&`, `<`), decimal (`á`), and hexadecimal (`&xE1;`) formats. Use for processing HTML content or decoding HTML-escaped data.", NewExampleSpec("", `root.unescaped = this.value.unescape_html()`, `{"value":"foo & bar"}`, `{"unescaped":"foo & bar"}`, ), + NewExampleSpec("", + `root.text = this.html.unescape_html()`, + `{"html":"<p>Hello & goodbye</p>"}`, + `{"text":"

Hello & goodbye

"}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return stringMethod(func(s string) (any, error) { @@ -579,12 +594,17 @@ var _ = registerSimpleMethod( "escape_url_query", "", ).InCategory( MethodCategoryStrings, - "Escapes a string so that it can be safely placed within a URL query.", + "Encodes a string for safe use in URL query parameters. Converts spaces to `+` and special characters to percent-encoded values. Use when building URLs with dynamic query parameters.", NewExampleSpec("", `root.escaped = this.value.escape_url_query()`, `{"value":"foo & bar"}`, `{"escaped":"foo+%26+bar"}`, ), + NewExampleSpec("", + `root.url = "https://example.com?search=" + this.query.escape_url_query()`, + `{"query":"hello world!"}`, + `{"url":"https://example.com?search=hello+world%21"}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return stringMethod(func(s string) (any, error) { @@ -600,12 +620,17 @@ var _ = registerSimpleMethod( "unescape_url_query", "", ).InCategory( MethodCategoryStrings, - "Expands escape sequences from a URL query string.", + "Decodes URL query parameter encoding, converting `+` to spaces and percent-encoded characters to their original values. Use for parsing URL query parameters.", NewExampleSpec("", `root.unescaped = this.value.unescape_url_query()`, `{"value":"foo+%26+bar"}`, `{"unescaped":"foo & bar"}`, ), + NewExampleSpec("", + `root.search = this.param.unescape_url_query()`, + `{"param":"hello+world%21"}`, + `{"search":"hello world!"}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return stringMethod(func(s string) (any, error) { @@ -621,7 +646,7 @@ var _ = registerSimpleMethod( "filepath_join", "", ).InCategory( MethodCategoryStrings, - "Joins an array of path elements into a single file path. The separator depends on the operating system of the machine.", + "Combines an array of path components into a single OS-specific file path using the correct separator (`/` on Unix, `\\` on Windows). Use for constructing file paths from components.", NewExampleSpec("", `root.path = this.path_elements.filepath_join()`, strings.ReplaceAll(`{"path_elements":["/foo/","bar.txt"]}`, "/", string(filepath.Separator)), @@ -652,7 +677,7 @@ var _ = registerSimpleMethod( "filepath_split", "", ).InCategory( MethodCategoryStrings, - "Splits a file path immediately following the final Separator, separating it into a directory and file name component returned as a two element array of strings. If there is no Separator in the path, the first element will be empty and the second will contain the path. The separator depends on the operating system of the machine.", + "Separates a file path into directory and filename components, returning a two-element array `[directory, filename]`. Use for extracting the filename or directory from a full path.", NewExampleSpec("", `root.path_sep = this.path.filepath_split()`, strings.ReplaceAll(`{"path":"/foo/bar.txt"}`, "/", string(filepath.Separator)), @@ -676,12 +701,17 @@ var _ = registerSimpleMethod( "format", "", ).InCategory( MethodCategoryStrings, - "Use a value string as a format specifier in order to produce a new string, using any number of provided arguments. Please refer to the Go https://pkg.go.dev/fmt[`fmt` package documentation^] for the list of valid format verbs.", + "Formats a string using Go's printf-style formatting with the string as the format template. Supports all Go format verbs (`%s`, `%d`, `%v`, etc.). Use for building formatted strings from dynamic values.", NewExampleSpec("", `root.foo = "%s(%v): %v".format(this.name, this.age, this.fingers)`, `{"name":"lance","age":37,"fingers":13}`, `{"foo":"lance(37): 13"}`, ), + NewExampleSpec("", + `root.message = "User %s has %v points".format(this.username, this.score)`, + `{"username":"alice","score":100}`, + `{"message":"User alice has 100 points"}`, + ), ).VariadicParams(), func(args *ParsedParams) (simpleMethod, error) { return stringMethod(func(s string) (any, error) { @@ -697,7 +727,7 @@ var _ = registerSimpleMethod( "has_prefix", "", ).InCategory( MethodCategoryStrings, - "Checks whether a string has a prefix argument and returns a bool.", + "Tests if a string starts with a specified prefix. Returns `true` if the string begins with the prefix, `false` otherwise. Use for conditional logic based on string patterns.", NewExampleSpec("", `root.t1 = this.v1.has_prefix("foo") root.t2 = this.v2.has_prefix("foo")`, @@ -730,7 +760,7 @@ var _ = registerSimpleMethod( "has_suffix", "", ).InCategory( MethodCategoryStrings, - "Checks whether a string has a suffix argument and returns a bool.", + "Tests if a string ends with a specified suffix. Returns `true` if the string ends with the suffix, `false` otherwise. Use for filtering or routing based on file extensions or string patterns.", NewExampleSpec("", `root.t1 = this.v1.has_suffix("foo") root.t2 = this.v2.has_suffix("foo")`, @@ -966,7 +996,7 @@ var _ = registerSimpleMethod( "join", "", ).InCategory( MethodCategoryObjectAndArray, - "Join an array of strings with an optional delimiter into a single string.", + "Concatenates an array of strings into a single string with an optional delimiter between elements. Use for building CSV strings, URLs, or combining text fragments.", NewExampleSpec("", `root.joined_words = this.words.join() root.joined_numbers = this.numbers.map_each(this.string()).join(",")`, @@ -1015,12 +1045,17 @@ var _ = registerSimpleMethod( "uppercase", "", ).InCategory( MethodCategoryStrings, - "Convert a string value into uppercase.", + "Converts all letters in a string to uppercase. Use for case-insensitive comparisons or formatting output.", NewExampleSpec("", `root.foo = this.foo.uppercase()`, `{"foo":"hello world"}`, `{"foo":"HELLO WORLD"}`, ), + NewExampleSpec("", + `root.code = this.product_code.uppercase()`, + `{"product_code":"abc-123"}`, + `{"code":"ABC-123"}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return func(v any, ctx FunctionContext) (any, error) { @@ -1043,12 +1078,17 @@ var _ = registerSimpleMethod( "lowercase", "", ).InCategory( MethodCategoryStrings, - "Convert a string value into lowercase.", + "Converts all letters in a string to lowercase. Use for case-insensitive comparisons, normalization, or formatting output.", NewExampleSpec("", `root.foo = this.foo.lowercase()`, `{"foo":"HELLO WORLD"}`, `{"foo":"hello world"}`, ), + NewExampleSpec("", + `root.email = this.user_email.lowercase()`, + `{"user_email":"User@Example.COM"}`, + `{"email":"user@example.com"}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return func(v any, ctx FunctionContext) (any, error) { @@ -1441,7 +1481,7 @@ var _ = registerSimpleMethod( "reverse", "", ).InCategory( MethodCategoryStrings, - "Returns the target string in reverse order.", + "Reverses the order of characters in a string. Unicode-aware for proper handling of multi-byte characters. Use for creating palindrome checks or reversing text data.", NewExampleSpec("", `root.reversed = this.thing.reverse()`, `{"thing":"backwards"}`, @@ -1482,12 +1522,17 @@ var _ = registerSimpleMethod( "quote", "", ).InCategory( MethodCategoryStrings, - "Quotes a target string using escape sequences (`\\t`, `\\n`, `\\xFF`, `\\u0100`) for control characters and non-printable characters.", + "Wraps a string in double quotes and escapes special characters (newlines, tabs, etc.) using Go escape sequences. Use for generating string literals or preparing strings for JSON-like formats.", NewExampleSpec("", `root.quoted = this.thing.quote()`, `{"thing":"foo\nbar"}`, `{"quoted":"\"foo\\nbar\""}`, ), + NewExampleSpec("", + `root.literal = this.text.quote()`, + `{"text":"hello\tworld"}`, + `{"literal":"\"hello\\tworld\""}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return stringMethod(func(s string) (any, error) { @@ -1503,12 +1548,17 @@ var _ = registerSimpleMethod( "unquote", "", ).InCategory( MethodCategoryStrings, - "Unquotes a target string, expanding any escape sequences (`\\t`, `\\n`, `\\xFF`, `\\u0100`) for control characters and non-printable characters.", + "Removes surrounding quotes and interprets escape sequences (`\\n`, `\\t`, etc.) to their literal characters. Use for parsing quoted string literals.", NewExampleSpec("", `root.unquoted = this.thing.unquote()`, `{"thing":"\"foo\\nbar\""}`, `{"unquoted":"foo\nbar"}`, ), + NewExampleSpec("", + `root.text = this.literal.unquote()`, + `{"literal":"\"hello\\tworld\""}`, + `{"text":"hello\tworld"}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return stringMethod(func(s string) (any, error) { @@ -1531,12 +1581,17 @@ var _ = registerSimpleMethod( "replace_all", "", ).InCategory( MethodCategoryStrings, - "Replaces all occurrences of the first argument in a target string with the second argument.", + "Replaces all occurrences of a substring with another string. Use for text transformation, cleaning data, or normalizing strings.", NewExampleSpec("", `root.new_value = this.value.replace_all("foo","dog")`, `{"value":"The foo ate my homework"}`, `{"new_value":"The dog ate my homework"}`, ), + NewExampleSpec("", + `root.clean = this.text.replace_all(" ", " ")`, + `{"text":"hello world foo"}`, + `{"clean":"hello world foo"}`, + ), ). Param(ParamString("old", "A string to match against.")). Param(ParamString("new", "A string to replace with.")), @@ -1577,7 +1632,7 @@ var _ = registerSimpleMethod( "replace_all_many", "", ).InCategory( MethodCategoryStrings, - "For each pair of strings in an argument array, replaces all occurrences of the first item of the pair with the second. This is a more compact way of chaining a series of `replace_all` methods.", + "Performs multiple find-and-replace operations in sequence using an array of `[old, new]` pairs. More efficient than chaining multiple `replace_all` calls. Use for bulk text transformations.", NewExampleSpec("", `root.new_value = this.value.replace_all_many([ "", "<b>", @@ -1641,12 +1696,17 @@ var _ = registerSimpleMethod( "re_find_all", "", ).InCategory( MethodCategoryRegexp, - "Returns an array containing all successive matches of a regular expression in a string.", + "Finds all matches of a regular expression in a string and returns them as an array. Use for extracting multiple patterns or validating repeating structures.", NewExampleSpec("", `root.matches = this.value.re_find_all("a.")`, `{"value":"paranormal"}`, `{"matches":["ar","an","al"]}`, ), + NewExampleSpec("", + `root.numbers = this.text.re_find_all("[0-9]+")`, + `{"text":"I have 2 apples and 15 oranges"}`, + `{"numbers":["2","15"]}`, + ), ).Param(ParamString("pattern", "The pattern to match against.")), func(args *ParsedParams) (simpleMethod, error) { reStr, err := args.FieldString("pattern") @@ -1687,12 +1747,17 @@ var _ = registerSimpleMethod( "re_find_all_submatch", "", ).InCategory( MethodCategoryRegexp, - "Returns an array of arrays containing all successive matches of the regular expression in a string and the matches, if any, of its subexpressions.", + "Finds all regex matches and their capture groups, returning an array of arrays where each inner array contains the full match and captured subgroups. Use for extracting structured data with capture groups.", NewExampleSpec("", `root.matches = this.value.re_find_all_submatch("a(x*)b")`, `{"value":"-axxb-ab-"}`, `{"matches":[["axxb","xx"],["ab",""]]}`, ), + NewExampleSpec("", + `root.emails = this.text.re_find_all_submatch("(\\w+)@(\\w+\\.\\w+)")`, + `{"text":"Contact: alice@example.com or bob@test.org"}`, + `{"emails":[["alice@example.com","alice","example.com"],["bob@test.org","bob","test.org"]]}`, + ), ).Param(ParamString("pattern", "The pattern to match against.")), func(args *ParsedParams) (simpleMethod, error) { reStr, err := args.FieldString("pattern") @@ -1741,7 +1806,7 @@ var _ = registerSimpleMethod( "re_find_object", "", ).InCategory( MethodCategoryRegexp, - "Returns an object containing the first match of the regular expression and the matches of its subexpressions. The key of each match value is the name of the group when specified, otherwise it is the index of the matching group, starting with the expression as a whole at 0.", + "Finds the first regex match and returns an object with named capture groups as keys (or numeric indices for unnamed groups). The key \"0\" contains the full match. Use for parsing structured text into fields.", NewExampleSpec("", `root.matches = this.value.re_find_object("a(?Px*)b")`, `{"value":"-axxb-ab-"}`, @@ -1798,7 +1863,7 @@ var _ = registerSimpleMethod( "re_find_all_object", "", ).InCategory( MethodCategoryRegexp, - "Returns an array of objects containing all matches of the regular expression and the matches of its subexpressions. The key of each match value is the name of the group when specified, otherwise it is the index of the matching group, starting with the expression as a whole at 0.", + "Finds all regex matches and returns an array of objects with named capture groups as keys. Each object represents one match with its captured groups. Use for parsing multiple structured records from text.", NewExampleSpec("", `root.matches = this.value.re_find_all_object("a(?Px*)b")`, `{"value":"-axxb-ab-"}`, @@ -1865,7 +1930,7 @@ var _ = registerSimpleMethod( "re_match", "", ).InCategory( MethodCategoryRegexp, - "Checks whether a regular expression matches against any part of a string and returns a boolean.", + "Tests if a regular expression matches anywhere in a string, returning `true` or `false`. Use for validation or conditional routing based on patterns.", NewExampleSpec("", `root.matches = this.value.re_match("[0-9]")`, `{"value":"there are 10 puppies"}`, @@ -1912,12 +1977,17 @@ var _ = registerSimpleMethod( "re_replace_all", "", ).InCategory( MethodCategoryRegexp, - "Replaces all occurrences of the argument regular expression in a string with a value. Inside the value $ signs are interpreted as submatch expansions, e.g. `$1` represents the text of the first submatch.", + "Replaces all regex matches with a replacement string that can reference capture groups using `$1`, `$2`, etc. Use for pattern-based transformations or data reformatting.", NewExampleSpec("", `root.new_value = this.value.re_replace_all("ADD ([0-9]+)","+($1)")`, `{"value":"foo ADD 70"}`, `{"new_value":"foo +(70)"}`, ), + NewExampleSpec("", + `root.masked = this.email.re_replace_all("(\\w{2})\\w+@", "$1***@")`, + `{"email":"alice@example.com"}`, + `{"masked":"al***@example.com"}`, + ), ). Param(ParamString("pattern", "The pattern to match against.")). Param(ParamString("value", "The value to replace with.")), @@ -1959,12 +2029,17 @@ var _ = registerSimpleMethod( "split", "", ).InCategory( MethodCategoryStrings, - "Split a string value into an array of strings by splitting it on a string separator.", + "Splits a string into an array of substrings using a delimiter. Use for parsing CSV-like data, splitting paths, or breaking text into tokens.", NewExampleSpec("", `root.new_value = this.value.split(",")`, `{"value":"foo,bar,baz"}`, `{"new_value":["foo","bar","baz"]}`, ), + NewExampleSpec("", + `root.words = this.sentence.split(" ")`, + `{"sentence":"hello world from bloblang"}`, + `{"words":["hello","world","from","bloblang"]}`, + ), ).Param(ParamString("delimiter", "The delimiter to split with.")), func(args *ParsedParams) (simpleMethod, error) { delim, err := args.FieldString("delimiter") @@ -2001,7 +2076,7 @@ var _ = registerSimpleMethod( "string", "", ).InCategory( MethodCategoryCoercion, - "Marshal a value into a string. If the value is already a string it is unchanged.", + "Converts any value to its string representation. Numbers, booleans, and objects are converted to strings; existing strings are unchanged. Use for type coercion or creating string representations.", NewExampleSpec("", `root.nested_json = this.string()`, `{"foo":"bar"}`, @@ -2027,7 +2102,7 @@ var _ = registerSimpleMethod( "trim", "", ).InCategory( MethodCategoryStrings, - "Remove all leading and trailing characters from a string that are contained within an argument cutset. If no arguments are provided then whitespace is removed.", + "Removes leading and trailing characters from a string. Without arguments, removes whitespace. With a cutset argument, removes any characters in the cutset. Use for cleaning user input or normalizing strings.", NewExampleSpec("", `root.title = this.title.trim("!?") root.description = this.description.trim()`, @@ -2063,7 +2138,7 @@ var _ = registerSimpleMethod( "trim_prefix", "", ).InCategory( MethodCategoryStrings, - "Remove the provided leading prefix substring from a string. If the string does not have the prefix substring, it is returned unchanged.", + "Removes a specified prefix from the beginning of a string if present. If the string doesn't start with the prefix, returns the string unchanged. Use for stripping known prefixes from identifiers or paths.", NewExampleSpec("", `root.name = this.name.trim_prefix("foobar_") root.description = this.description.trim_prefix("foobar_")`, @@ -2095,7 +2170,7 @@ var _ = registerSimpleMethod( "trim_suffix", "", ).InCategory( MethodCategoryStrings, - "Remove the provided trailing suffix substring from a string. If the string does not have the suffix substring, it is returned unchanged.", + "Removes a specified suffix from the end of a string if present. If the string doesn't end with the suffix, returns the string unchanged. Use for stripping file extensions or known suffixes.", NewExampleSpec("", `root.name = this.name.trim_suffix("_foobar") root.description = this.description.trim_suffix("_foobar")`, @@ -2129,13 +2204,18 @@ var _ = registerSimpleMethod( "repeat", "", ).InCategory( MethodCategoryStrings, - "Repeat returns a new string consisting of count copies of the string", + "Creates a new string by repeating the input string a specified number of times. Use for generating padding, separators, or test data.", NewExampleSpec("", `root.repeated = this.name.repeat(3) root.not_repeated = this.name.repeat(0)`, `{"name":"bob"}`, `{"not_repeated":"","repeated":"bobbobbob"}`, ), + NewExampleSpec("", + `root.separator = "-".repeat(10)`, + `{}`, + `{"separator":"----------"}`, + ), ).Param(ParamInt64("count", "The number of times to repeat the string.")), func(args *ParsedParams) (simpleMethod, error) { count, err := args.FieldInt64("count") diff --git a/internal/bloblang/query/methods_structured.go b/internal/bloblang/query/methods_structured.go index 9b05b8bcd..0c6a6f49c 100644 --- a/internal/bloblang/query/methods_structured.go +++ b/internal/bloblang/query/methods_structured.go @@ -19,7 +19,7 @@ import ( var _ = registerSimpleMethod( NewMethodSpec( "all", - "Checks each element of an array against a query and returns true if all elements passed. An error occurs if the target is not an array, or if any element results in the provided query returning a non-boolean result. Returns false if the target array is empty.", + "Tests whether all elements in an array satisfy a condition. Returns true only if the query evaluates to true for every element. Returns false for empty arrays.", ).InCategory( MethodCategoryObjectAndArray, "", @@ -30,6 +30,13 @@ var _ = registerSimpleMethod( `{"patrons":[{"id":"1","age":45},{"id":"2","age":23}]}`, `{"all_over_21":true}`, ), + NewExampleSpec("", + `root.all_positive = this.values.all(v -> v > 0)`, + `{"values":[1,2,3,4,5]}`, + `{"all_positive":true}`, + `{"values":[1,-2,3,4,5]}`, + `{"all_positive":false}`, + ), ).Param(ParamQuery("test", "A test query to apply to each element.", false)), func(args *ParsedParams) (simpleMethod, error) { queryFn, err := args.FieldQuery("test") @@ -65,7 +72,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( "any", - "Checks the elements of an array against a query and returns true if any element passes. An error occurs if the target is not an array, or if an element results in the provided query returning a non-boolean result. Returns false if the target array is empty.", + "Tests whether at least one element in an array satisfies a condition. Returns true if the query evaluates to true for any element. Returns false for empty arrays.", ).InCategory( MethodCategoryObjectAndArray, "", @@ -76,6 +83,13 @@ var _ = registerSimpleMethod( `{"patrons":[{"id":"1","age":10},{"id":"2","age":12}]}`, `{"any_over_21":false}`, ), + NewExampleSpec("", + `root.has_errors = this.results.any(r -> r.status == "error")`, + `{"results":[{"status":"ok"},{"status":"error"},{"status":"ok"}]}`, + `{"has_errors":true}`, + `{"results":[{"status":"ok"},{"status":"ok"}]}`, + `{"has_errors":false}`, + ), ).Param(ParamQuery("test", "A test query to apply to each element.", false)), func(args *ParsedParams) (simpleMethod, error) { queryFn, err := args.FieldQuery("test") @@ -116,7 +130,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( "append", - "Returns an array with new elements appended to the end.", + "Adds one or more elements to the end of an array and returns the new array. The original array is not modified.", ).InCategory( MethodCategoryObjectAndArray, "", @@ -125,6 +139,11 @@ var _ = registerSimpleMethod( `{"foo":["bar","baz"]}`, `{"foo":["bar","baz","and","this"]}`, ), + NewExampleSpec("", + `root.combined = this.items.append(this.new_item)`, + `{"items":["apple","banana"],"new_item":"orange"}`, + `{"combined":["apple","banana","orange"]}`, + ), ).VariadicParams(), func(args *ParsedParams) (simpleMethod, error) { argsList := args.Raw() @@ -147,14 +166,14 @@ var _ = registerSimpleMethod( "collapse", "", ).InCategory( MethodCategoryObjectAndArray, - "Collapse an array or object into an object of key/value pairs for each field, where the key is the full path of the structured field in dot path notation. Empty arrays an objects are ignored by default.", + "Flattens a nested structure into a single-level object with dot-notation keys representing the full path to each value. Empty arrays and objects are excluded by default.", NewExampleSpec("", `root.result = this.collapse()`, `{"foo":[{"bar":"1"},{"bar":{}},{"bar":"2"},{"bar":[]}]}`, `{"result":{"foo.0.bar":"1","foo.2.bar":"2"}}`, ), NewExampleSpec( - "An optional boolean parameter can be set to true in order to include empty objects and arrays.", + "Set include_empty to true to preserve empty objects and arrays in the output.", `root.result = this.collapse(include_empty: true)`, `{"foo":[{"bar":"1"},{"bar":{}},{"bar":"2"},{"bar":[]}]}`, `{"result":{"foo.0.bar":"1","foo.1.bar":{},"foo.2.bar":"2","foo.3.bar":[]}}`, @@ -246,7 +265,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( "enumerated", - "Converts an array into a new array of objects, where each object has a field index containing the `index` of the element and a field `value` containing the original value of the element.", + "Transforms an array into an array of objects with index and value fields, making it easy to access both the position and content of each element.", ).InCategory( MethodCategoryObjectAndArray, "", NewExampleSpec("", @@ -254,6 +273,11 @@ var _ = registerSimpleMethod( `{"foo":["bar","baz"]}`, `{"foo":[{"index":0,"value":"bar"},{"index":1,"value":"baz"}]}`, ), + NewExampleSpec("Useful for filtering by index position", + `root.first_two = this.items.enumerated().filter(item -> item.index < 2).map_each(item -> item.value)`, + `{"items":["a","b","c","d"]}`, + `{"first_two":["a","b"]}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return func(v any, ctx FunctionContext) (any, error) { @@ -278,7 +302,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( "exists", - "Checks that a field, identified via a xref:configuration:field_paths.adoc[dot path], exists in an object.", + "Checks whether a field exists at the specified dot path within an object. Returns true if the field is present (even if null), false otherwise.", NewExampleSpec("", `root.result = this.foo.exists("bar.baz")`, `{"foo":{"bar":{"baz":"yep, I exist"}}}`, @@ -288,6 +312,13 @@ var _ = registerSimpleMethod( `{"foo":{}}`, `{"result":false}`, ), + NewExampleSpec("Also returns true for null values if the field exists", + `root.has_field = this.data.exists("optional_field")`, + `{"data":{"optional_field":null}}`, + `{"has_field":true}`, + `{"data":{}}`, + `{"has_field":false}`, + ), ).Param(ParamString("path", "A xref:configuration:field_paths.adoc[dot path] to a field.")), func(args *ParsedParams) (simpleMethod, error) { pathStr, err := args.FieldString("path") @@ -308,17 +339,17 @@ var _ = registerSimpleMethod( "explode", "", ).InCategory( MethodCategoryObjectAndArray, - "Explodes an array or object at a xref:configuration:field_paths.adoc[field path].", + "Expands a nested array or object field into multiple documents, distributing elements while preserving the surrounding structure. Useful for denormalizing data.", NewExampleSpec(`##### On arrays -Exploding arrays results in an array containing elements matching the original document, where the target field of each element is an element of the exploded array:`, +When exploding an array, each element becomes a separate document with the array element replacing the original field:`, `root = this.explode("value")`, `{"id":1,"value":["foo","bar","baz"]}`, `[{"id":1,"value":"foo"},{"id":1,"value":"bar"},{"id":1,"value":"baz"}]`, ), NewExampleSpec(`##### On objects -Exploding objects results in an object where the keys match the target object, and the values match the original document but with the target field replaced by the exploded value:`, +When exploding an object, the output keys match the nested object's keys, with values being the full document where the target field is replaced by each nested value:`, `root = this.explode("value")`, `{"id":1,"value":{"foo":2,"bar":[3,4],"baz":{"bev":5}}}`, `{"bar":{"id":1,"value":[3,4]},"baz":{"id":1,"value":{"bev":5}},"foo":{"id":1,"value":2}}`, @@ -370,7 +401,7 @@ var _ = registerSimpleMethod( "filter", "", ).InCategory( MethodCategoryObjectAndArray, - "Executes a mapping query argument for each element of an array or key/value pair of an object. If the query returns `false` the item is removed from the resulting array or object. The item will also be removed if the query returns any non-boolean value.", + "Returns a new array or object containing only elements that satisfy the provided condition. Elements for which the query returns true are kept, all others are removed.", NewExampleSpec(``, `root.new_nums = this.nums.filter(num -> num > 10)`, `{"nums":[3,11,4,17]}`, @@ -378,7 +409,7 @@ var _ = registerSimpleMethod( ), NewExampleSpec(`##### On objects -When filtering objects the mapping query argument is provided a context with a field `+"`key`"+` containing the value key, and a field `+"`value`"+` containing the value.`, +When filtering objects, the query receives a context with `+"`key`"+` and `+"`value`"+` fields for each entry:`, `root.new_dict = this.dict.filter(item -> item.value.contains("foo"))`, `{"dict":{"first":"hello foo","second":"world","third":"this foo is great"}}`, `{"new_dict":{"first":"hello foo","third":"this foo is great"}}`, @@ -433,7 +464,7 @@ When filtering objects the mapping query argument is provided a context with a f var _ = registerSimpleMethod( NewMethodSpec( "find", - "Returns the index of the first occurrence of a value in an array. `-1` is returned if there are no matches. Numerical comparisons are made irrespective of the representation type (float versus integer).", + "Searches an array for a matching value and returns the index of the first occurrence. Returns -1 if no match is found. Numeric types are compared by value regardless of representation.", ).InCategory( MethodCategoryObjectAndArray, "", NewExampleSpec("", @@ -474,7 +505,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( "find_all", - "Returns an array containing the indexes of all occurrences of a value in an array. An empty array is returned if there are no matches. Numerical comparisons are made irrespective of the representation type (float versus integer).", + "Searches an array for all occurrences of a value and returns an array of matching indexes. Returns an empty array if no matches are found. Numeric types are compared by value regardless of representation.", ).InCategory( MethodCategoryObjectAndArray, "", NewExampleSpec("", @@ -517,7 +548,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( "find_by", - "Returns the index of the first occurrence of an array where the provided query resolves to a boolean `true`. `-1` is returned if there are no matches.", + "Searches an array for the first element that satisfies a condition and returns its index. Returns -1 if no element matches the query.", ).InCategory( MethodCategoryObjectAndArray, "", NewExampleSpec("", @@ -525,6 +556,11 @@ var _ = registerSimpleMethod( `["foo", "bar", "baz"]`, `{"index":0}`, ), + NewExampleSpec("Find first object matching criteria", + `root.first_adult = this.users.find_by(u -> u.age >= 18)`, + `{"users":[{"name":"Alice","age":15},{"name":"Bob","age":22},{"name":"Carol","age":19}]}`, + `{"first_adult":1}`, + ), ).Beta().Param(ParamQuery("query", "A query to execute for each element.", false)), func(args *ParsedParams) (simpleMethod, error) { queryFn, err := args.FieldQuery("query") @@ -561,7 +597,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( "find_all_by", - "Returns an array containing the indexes of all occurrences of an array where the provided query resolves to a boolean `true`. An empty array is returned if there are no matches. Numerical comparisons are made irrespective of the representation type (float versus integer).", + "Searches an array for all elements that satisfy a condition and returns an array of their indexes. Returns an empty array if no elements match.", ).InCategory( MethodCategoryObjectAndArray, "", NewExampleSpec("", @@ -569,6 +605,11 @@ var _ = registerSimpleMethod( `["foo", "bar", "baz"]`, `{"index":[0,2]}`, ), + NewExampleSpec("Find all indexes matching criteria", + `root.error_indexes = this.logs.find_all_by(log -> log.level == "error")`, + `{"logs":[{"level":"info"},{"level":"error"},{"level":"warn"},{"level":"error"}]}`, + `{"error_indexes":[1,3]}`, + ), ).Beta().Param(ParamQuery("query", "A query to execute for each element.", false)), func(args *ParsedParams) (simpleMethod, error) { queryFn, err := args.FieldQuery("query") @@ -607,7 +648,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( "flatten", - "Iterates an array and any element that is itself an array is removed and has its elements inserted directly in the resulting array.", + "Flattens an array by one level, expanding nested arrays into the parent array. Only the first level of nesting is removed.", ).InCategory( MethodCategoryObjectAndArray, "", NewExampleSpec(``, @@ -615,6 +656,11 @@ var _ = registerSimpleMethod( `["foo",["bar","baz"],"buz"]`, `{"result":["foo","bar","baz","buz"]}`, ), + NewExampleSpec("Deeper nesting requires multiple flatten calls", + `root.result = this.data.flatten()`, + `{"data":["a",["b",["c","d"]],"e"]}`, + `{"result":["a","b",["c","d"],"e"]}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return func(v any, ctx FunctionContext) (any, error) { @@ -641,20 +687,20 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( "fold", - "Takes two arguments: an initial value, and a mapping query. For each element of an array the mapping context is an object with two fields `tally` and `value`, where `tally` contains the current accumulated value and `value` is the value of the current element. The mapping must return the result of adding the value to the tally.\n\nThe first argument is the value that `tally` will have on the first call.", + "Reduces an array to a single value by iteratively applying a function. Also known as reduce or aggregate. The query receives an accumulator (tally) and current element (value) for each iteration.", ).InCategory( MethodCategoryObjectAndArray, "", - NewExampleSpec(``, + NewExampleSpec(`Sum numbers in an array`, `root.sum = this.foo.fold(0, item -> item.tally + item.value)`, `{"foo":[3,8,11]}`, `{"sum":22}`, ), - NewExampleSpec(``, + NewExampleSpec(`Concatenate strings`, `root.result = this.foo.fold("", item -> "%v%v".format(item.tally, item.value))`, `{"foo":["hello ", "world"]}`, `{"result":"hello world"}`, ), - NewExampleSpec(`You can use fold to merge an array of objects together:`, + NewExampleSpec(`Merge an array of objects into a single object`, `root.smoothie = this.fruits.fold({}, item -> item.tally.merge(item.value))`, `{"fruits":[{"apple":5},{"banana":3},{"orange":8}]}`, `{"smoothie":{"apple":5,"banana":3,"orange":8}}`, @@ -810,7 +856,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( "keys", - "Returns the keys of an object as an array.", + "Extracts all keys from an object and returns them as a sorted array.", ).InCategory( MethodCategoryObjectAndArray, "", NewExampleSpec("", @@ -818,6 +864,11 @@ var _ = registerSimpleMethod( `{"foo":{"bar":1,"baz":2}}`, `{"foo_keys":["bar","baz"]}`, ), + NewExampleSpec("Check if specific keys exist", + `root.has_id = this.data.keys().contains("id")`, + `{"data":{"id":123,"name":"test"}}`, + `{"has_id":true}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return func(v any, ctx FunctionContext) (any, error) { @@ -839,7 +890,7 @@ var _ = registerSimpleMethod( var _ = registerSimpleMethod( NewMethodSpec( "key_values", - "Returns the key/value pairs of an object as an array, where each element is an object with a `key` field and a `value` field. The order of the resulting array will be random.", + "Converts an object into an array of key-value pair objects. Each element has a 'key' field and a 'value' field. Order is not guaranteed unless sorted.", ).InCategory( MethodCategoryObjectAndArray, "", NewExampleSpec("", @@ -848,6 +899,11 @@ var _ = registerSimpleMethod( `{"foo":{"bar":1,"baz":2}}`, `{"foo_key_values":[{"key":"bar","value":1},{"key":"baz","value":2}]}`, ), + NewExampleSpec("Filter object entries by value", + `root.large_items = this.items.key_values().filter(pair -> pair.value > 15).map_each(pair -> pair.key)`, + `{"items":{"a":5,"b":15,"c":20,"d":3}}`, + `{"large_items":["c"]}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return func(v any, ctx FunctionContext) (any, error) { @@ -872,14 +928,14 @@ var _ = registerSimpleMethod( NewMethodSpec( "length", "", ).InCategory( - MethodCategoryStrings, "Returns the length of a string.", + MethodCategoryStrings, "Returns the character count of a string.", NewExampleSpec("", `root.foo_len = this.foo.length()`, `{"foo":"hello world"}`, `{"foo_len":11}`, ), ).InCategory( - MethodCategoryObjectAndArray, "Returns the length of an array or object (number of keys).", + MethodCategoryObjectAndArray, "Returns the size of an array (element count) or object (key count).", NewExampleSpec("", `root.foo_len = this.foo.length()`, `{"foo":["first","second"]}`, @@ -917,7 +973,7 @@ var _ = registerSimpleMethod( MethodCategoryObjectAndArray, "", NewExampleSpec(`##### On arrays -Apply a mapping to each element of an array and replace the element with the result. Within the argument mapping the context is the value of the element being mapped.`, +Transforms each array element using a query. Return deleted() to remove an element, or the new value to replace it.`, `root.new_nums = this.nums.map_each(num -> if num < 10 { deleted() } else { @@ -928,7 +984,7 @@ Apply a mapping to each element of an array and replace the element with the res ), NewExampleSpec(`##### On objects -Apply a mapping to each value of an object and replace the value with the result. Within the argument mapping the context is an object with a field `+"`key`"+` containing the value key, and a field `+"`value`"+`.`, +Transforms each object value using a query. The query receives an object with 'key' and 'value' fields for each entry.`, `root.new_dict = this.dict.map_each(item -> item.value.uppercase())`, `{"dict":{"foo":"hello","bar":"world"}}`, `{"new_dict":{"bar":"WORLD","foo":"HELLO"}}`, @@ -996,13 +1052,13 @@ var _ = registerSimpleMethod( NewMethodSpec( "map_each_key", "", ).InCategory( - MethodCategoryObjectAndArray, `Apply a mapping to each key of an object, and replace the key with the result, which must be a string.`, + MethodCategoryObjectAndArray, `Transforms object keys using a query. The query receives each key as a string and must return a new string key. Use this to rename or transform keys while preserving values.`, NewExampleSpec(``, `root.new_dict = this.dict.map_each_key(key -> key.uppercase())`, `{"dict":{"keya":"hello","keyb":"world"}}`, `{"new_dict":{"KEYA":"hello","KEYB":"world"}}`, ), - NewExampleSpec(``, + NewExampleSpec(`Conditionally transform keys`, `root = this.map_each_key(key -> if key.contains("kafka") { "_" + key })`, `{"amqp_key":"foo","kafka_key":"bar","kafka_topic":"baz"}`, `{"_kafka_key":"bar","_kafka_topic":"baz","amqp_key":"foo"}`, @@ -1047,7 +1103,7 @@ var _ = registerSimpleMethod( var _ = registerMethod( NewMethodSpec( - "merge", "Merge a source object into an existing destination object. When a collision is found within the merged structures (both a source and destination object contain the same non-object keys) the result will be an array containing both values, where values that are already arrays will be expanded into the resulting array. In order to simply override destination fields on collision use the <> method.", + "merge", "Combines two objects or arrays. When merging objects, conflicting keys create arrays containing both values. Arrays are concatenated. For key override behavior instead, use the assign method.", ).InCategory( MethodCategoryObjectAndArray, "", NewExampleSpec(``, @@ -1055,6 +1111,11 @@ var _ = registerMethod( `{"foo":{"first_name":"fooer","likes":"bars"},"bar":{"second_name":"barer","likes":"foos"}}`, `{"first_name":"fooer","likes":["bars","foos"],"second_name":"barer"}`, ), + NewExampleSpec(`Merge arrays`, + `root.combined = this.list1.merge(this.list2)`, + `{"list1":["a","b"],"list2":["c","d"]}`, + `{"combined":["a","b","c","d"]}`, + ), ).Param(ParamAny("with", "A value to merge the target value with.")), mergeMethod, ) @@ -1097,7 +1158,7 @@ func mergeMethod(target Function, args *ParsedParams) (Function, error) { var _ = registerMethod( NewMethodSpec( - "assign", "Merge a source object into an existing destination object. When a collision is found within the merged structures (both a source and destination object contain the same non-object keys) the value in the destination object will be overwritten by that of source object. In order to preserve both values on collision use the <> method.", + "assign", "Merges two objects or arrays with override behavior. For objects, source values replace destination values on key conflicts. Arrays are concatenated. To preserve both values on conflict, use the merge method instead.", ).InCategory( MethodCategoryObjectAndArray, "", NewExampleSpec(``, @@ -1105,6 +1166,11 @@ var _ = registerMethod( `{"foo":{"first_name":"fooer","likes":"bars"},"bar":{"second_name":"barer","likes":"foos"}}`, `{"first_name":"fooer","likes":"foos","second_name":"barer"}`, ), + NewExampleSpec(`Override defaults with user settings`, + `root.config = this.defaults.assign(this.user_settings)`, + `{"defaults":{"timeout":30,"retries":3},"user_settings":{"timeout":60}}`, + `{"config":{"retries":3,"timeout":60}}`, + ), ).Param(ParamAny("with", "A value to merge the target value with.")), assignMethod, ) @@ -1206,13 +1272,13 @@ var _ = registerMethod( "sort", "", ).InCategory( MethodCategoryObjectAndArray, - "Attempts to sort the values of an array in increasing order. The type of all values must match in order for the ordering to succeed. Supports string and number values.", + "Sorts an array in ascending order. Works with strings and numbers. For custom sorting logic, provide a comparison query that receives 'left' and 'right' elements.", NewExampleSpec("", `root.sorted = this.foo.sort()`, `{"foo":["bbb","ccc","aaa"]}`, `{"sorted":["aaa","bbb","ccc"]}`, ), - NewExampleSpec("It's also possible to specify a mapping argument, which is provided an object context with fields `left` and `right`, the mapping must return a boolean indicating whether the `left` value is less than `right`. This allows you to sort arrays containing non-string or non-number values.", + NewExampleSpec("Custom comparison for complex objects - return true if left < right", `root.sorted = this.foo.sort(item -> item.left.v < item.right.v)`, `{"foo":[{"id":"foo","v":"bbb"},{"id":"bar","v":"ccc"},{"id":"baz","v":"aaa"}]}`, `{"sorted":[{"id":"baz","v":"aaa"},{"id":"foo","v":"bbb"},{"id":"bar","v":"ccc"}]}`, @@ -1312,12 +1378,17 @@ var _ = registerMethod( "sort_by", "", ).InCategory( MethodCategoryObjectAndArray, - "Attempts to sort the elements of an array, in increasing order, by a value emitted by an argument query applied to each element. The type of all values must match in order for the ordering to succeed. Supports string and number values.", + "Sorts an array by a value extracted from each element using a query. The extracted values determine sort order and must all be strings or numbers.", NewExampleSpec("", `root.sorted = this.foo.sort_by(ele -> ele.id)`, `{"foo":[{"id":"bbb","message":"bar"},{"id":"aaa","message":"foo"},{"id":"ccc","message":"baz"}]}`, `{"sorted":[{"id":"aaa","message":"foo"},{"id":"bbb","message":"bar"},{"id":"ccc","message":"baz"}]}`, ), + NewExampleSpec("Sort by numeric field", + `root.sorted = this.items.sort_by(item -> item.priority)`, + `{"items":[{"name":"low","priority":3},{"name":"high","priority":1},{"name":"med","priority":2}]}`, + `{"sorted":[{"name":"high","priority":1},{"name":"med","priority":2},{"name":"low","priority":3}]}`, + ), ).Param(ParamQuery("query", "A query to apply to each element that yields a value used for sorting.", false)), sortByMethod, ) @@ -1503,12 +1574,17 @@ var _ = registerMethod( "sum", "", ).InCategory( MethodCategoryObjectAndArray, - "Sum the numerical values of an array.", + "Calculates the sum of all numeric values in an array. Non-numeric values cause an error.", NewExampleSpec("", `root.sum = this.foo.sum()`, `{"foo":[3,8,4]}`, `{"sum":15}`, ), + NewExampleSpec("Works with decimals", + `root.total = this.prices.sum()`, + `{"prices":[10.5,20.25,5.00]}`, + `{"total":35.75}`, + ), ), sumMethod, ) @@ -1548,12 +1624,17 @@ var _ = registerSimpleMethod( "unique", "", ).InCategory( MethodCategoryObjectAndArray, - "Attempts to remove duplicate values from an array. The array may contain a combination of different value types, but numbers and strings are checked separately (`\"5\"` is a different element to `5`).", + "Removes duplicate values from an array, keeping the first occurrence of each unique value. Strings and numbers are treated as distinct types (\"5\" differs from 5).", NewExampleSpec("", `root.uniques = this.foo.unique()`, `{"foo":["a","b","a","c"]}`, `{"uniques":["a","b","c"]}`, ), + NewExampleSpec("Use a query to determine uniqueness by a field", + `root.unique_users = this.users.unique(u -> u.id)`, + `{"users":[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"},{"id":1,"name":"Alice Duplicate"}]}`, + `{"unique_users":[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"}]}`, + ), ). Param(ParamQuery( "emit", @@ -1650,12 +1731,17 @@ var _ = registerSimpleMethod( "values", "", ).InCategory( MethodCategoryObjectAndArray, - "Returns the values of an object as an array. The order of the resulting array will be random.", + "Extracts all values from an object and returns them as an array. Order is not guaranteed unless the result is sorted.", NewExampleSpec("", `root.foo_vals = this.foo.values().sort()`, `{"foo":{"bar":1,"baz":2}}`, `{"foo_vals":[1,2]}`, ), + NewExampleSpec("Find max value in object", + `root.max = this.scores.values().sort().index(-1)`, + `{"scores":{"player1":85,"player2":92,"player3":78}}`, + `{"max":92}`, + ), ), func(*ParsedParams) (simpleMethod, error) { return func(v any, ctx FunctionContext) (any, error) { @@ -1678,14 +1764,17 @@ var _ = registerSimpleMethod( "without", "", ).InCategory( MethodCategoryObjectAndArray, - `Returns an object where one or more xref:configuration:field_paths.adoc[field path] arguments are removed. Each path specifies a specific field to be deleted from the input object, allowing for nested fields. - -If a key within a nested path does not exist or is not an object then it is not removed.`, + `Removes specified fields from an object using dot-notation paths. Returns a new object with the fields removed. Non-existent paths are safely ignored.`, NewExampleSpec("", `root = this.without("inner.a","inner.c","d")`, `{"inner":{"a":"first","b":"second","c":"third"},"d":"fourth","e":"fifth"}`, `{"e":"fifth","inner":{"b":"second"}}`, ), + NewExampleSpec("Remove sensitive fields", + `root = this.without("password","ssn","creditCard")`, + `{"username":"alice","password":"secret","email":"alice@example.com","ssn":"123-45-6789"}`, + `{"email":"alice@example.com","username":"alice"}`, + ), ).VariadicParams(), func(args *ParsedParams) (simpleMethod, error) { excludeList := make([][]string, 0, len(args.Raw())) diff --git a/internal/impl/io/bloblang.go b/internal/impl/io/bloblang.go index 98a5f7a7a..f19194b16 100644 --- a/internal/impl/io/bloblang.go +++ b/internal/impl/io/bloblang.go @@ -14,8 +14,8 @@ func init() { bloblang.NewPluginSpec(). Impure(). Category(query.FunctionCategoryEnvironment). - Description(`Returns a string matching the hostname of the machine running Benthos.`). - Example("", `root.thing.host = hostname()`), + Description(`Returns the hostname of the machine running Benthos. Useful for identifying which instance processed a message in distributed deployments.`). + ExampleNotTested("", `root.processed_by = hostname()`), func(_ *bloblang.ParsedParams) (bloblang.Function, error) { return func() (any, error) { hn, err := os.Hostname() @@ -35,17 +35,17 @@ func init() { return !noCache }). Category(query.FunctionCategoryEnvironment). - Description("Returns the value of an environment variable, or `null` if the environment variable does not exist."). + Description("Reads an environment variable and returns its value as a string. Returns `null` if the variable is not set. By default, values are cached for performance."). Param(bloblang.NewStringParam("name"). - Description("The name of an environment variable.")). + Description("The name of the environment variable to read.")). Param(bloblang.NewBoolParam("no_cache"). - Description("Force the variable lookup to occur for each mapping invocation."). + Description("Disable caching to read the latest value on each invocation."). Default(false)). - Example("", `root.thing.key = env("key").or("default value")`). - Example("", `root.thing.key = env(this.thing.key_name)`). - Example( - "When the name parameter is static this function will only resolve once and yield the same result for each invocation as an optimization, this means that updates to env vars during runtime will not be reflected. You can disable this cache with the optional parameter `no_cache`, which when set to `true` will cause the variable lookup to be performed for each execution of the mapping.", - `root.thing.key = env(name: "key", no_cache: true)`, + ExampleNotTested("", `root.api_key = env("API_KEY")`). + ExampleNotTested("", `root.database_url = env("DB_URL").or("localhost:5432")`). + ExampleNotTested( + "Use `no_cache` to read updated environment variables during runtime, useful for dynamic configuration changes.", + `root.config = env(name: "DYNAMIC_CONFIG", no_cache: true)`, ), func(args *bloblang.ParsedParams) (bloblang.Function, error) { name, err := args.GetString("name") @@ -85,20 +85,17 @@ func init() { return !noCache }). Category(query.FunctionCategoryEnvironment). - Description("Reads a file and returns its contents. Relative paths are resolved from the directory of the process executing the mapping. In order to read files relative to the mapping file use the newer <>"). + Description("Reads a file and returns its contents as bytes. Paths are resolved from the process working directory. For paths relative to the mapping file, use `file_rel`. By default, files are cached after first read."). Param(bloblang.NewStringParam("path"). - Description("The path of the target file.")). + Description("The absolute or relative path to the file.")). Param(bloblang.NewBoolParam("no_cache"). - Description("Force the file to be read for each mapping invocation."). + Description("Disable caching to read the latest file contents on each invocation."). Default(false)). - Example("", `root.doc = file(env("BENTHOS_TEST_BLOBLANG_FILE")).parse_json()`, [2]string{ - `{}`, - `{"doc":{"foo":"bar"}}`, - }). - Example( - "When the path parameter is static this function will only read the specified file once and yield the same result for each invocation as an optimization, this means that updates to files during runtime will not be reflected. You can disable this cache with the optional parameter `no_cache`, which when set to `true` will cause the file to be read for each execution of the mapping.", - `root.doc = file(path: env("BENTHOS_TEST_BLOBLANG_FILE"), no_cache: true).parse_json()`, - [2]string{`{}`, `{"doc":{"foo":"bar"}}`}, + ExampleNotTested("", `root.config = file("/etc/config.json").parse_json()`). + ExampleNotTested("", `root.template = file("./templates/email.html").string()`). + ExampleNotTested( + "Use `no_cache` to read updated file contents during runtime, useful for hot-reloading configuration.", + `root.rules = file(path: "/etc/rules.yaml", no_cache: true).parse_yaml()`, ), func(args *bloblang.ParsedParams) (bloblang.Function, error) { path, err := args.GetString("path") @@ -136,20 +133,17 @@ func init() { return !noCache }). Category(query.FunctionCategoryEnvironment). - Description("Reads a file and returns its contents. Relative paths are resolved from the directory of the mapping."). + Description("Reads a file and returns its contents as bytes. Paths are resolved relative to the mapping file's directory, making it portable across different environments. By default, files are cached after first read."). Param(bloblang.NewStringParam("path"). - Description("The path of the target file.")). + Description("The path to the file, relative to the mapping file's directory.")). Param(bloblang.NewBoolParam("no_cache"). - Description("Force the file to be read for each mapping invocation."). + Description("Disable caching to read the latest file contents on each invocation."). Default(false)). - Example("", `root.doc = file_rel(env("BENTHOS_TEST_BLOBLANG_FILE")).parse_json()`, [2]string{ - `{}`, - `{"doc":{"foo":"bar"}}`, - }). - Example( - "When the path parameter is static this function will only read the specified file once and yield the same result for each invocation as an optimization, this means that updates to files during runtime will not be reflected. You can disable this cache with the optional parameter `no_cache`, which when set to `true` will cause the file to be read for each execution of the mapping.", - `root.doc = file_rel(path: env("BENTHOS_TEST_BLOBLANG_FILE"), no_cache: true).parse_json()`, - [2]string{`{}`, `{"doc":{"foo":"bar"}}`}, + ExampleNotTested("", `root.schema = file_rel("./schemas/user.json").parse_json()`). + ExampleNotTested("", `root.lookup = file_rel("../data/lookup.csv").parse_csv()`). + ExampleNotTested( + "Use `no_cache` to read updated file contents during runtime, useful for reloading data files without restarting.", + `root.translations = file_rel(path: "./i18n/en.yaml", no_cache: true).parse_yaml()`, ), func(args *bloblang.ParsedParams) (bloblang.Function, error) { path, err := args.GetString("path") diff --git a/internal/impl/pure/bloblang_encoding.go b/internal/impl/pure/bloblang_encoding.go index 2f661d941..5f14dd6a8 100644 --- a/internal/impl/pure/bloblang_encoding.go +++ b/internal/impl/pure/bloblang_encoding.go @@ -11,22 +11,21 @@ func init() { bloblang.MustRegisterMethodV2("compress", bloblang.NewPluginSpec(). Category(query.MethodCategoryEncoding). - Description(`Compresses a string or byte array value according to a specified algorithm.`). - Param(bloblang.NewStringParam("algorithm").Description("One of `flate`, `gzip`, `pgzip`, `lz4`, `snappy`, `zlib`, `zstd`.")). - Param(bloblang.NewInt64Param("level").Description("The level of compression to use. May not be applicable to all algorithms.").Default(-1)). - Example("", `let long_content = range(0, 1000).map_each(content()).join(" ") -root.a_len = $long_content.length() -root.b_len = $long_content.compress("gzip").length() -`, + Description(`Compresses a string or byte array using the specified compression algorithm. Returns compressed data as bytes. Useful for reducing payload size before transmission or storage.`). + Param(bloblang.NewStringParam("algorithm").Description("The compression algorithm: `flate`, `gzip`, `pgzip` (parallel gzip), `lz4`, `snappy`, `zlib`, or `zstd`.")). + Param(bloblang.NewInt64Param("level").Description("Compression level (default: -1 for default compression). Higher values increase compression ratio but use more CPU. Range and effect varies by algorithm.").Default(-1)). + Example("Compress and encode for safe transmission", `root.compressed = content().bytes().compress("gzip").encode("base64")`, [2]string{ - `hello world this is some content`, - `{"a_len":32999,"b_len":161}`, + `{"message":"hello world I love space"}`, + `{"compressed":"H4sIAAAJbogA/wAmANn/eyJtZXNzYWdlIjoiaGVsbG8gd29ybGQgSSBsb3ZlIHNwYWNlIn0DAHEvdwomAAAA"}`, }, ). - Example("", `root.compressed = content().compress("lz4").encode("base64")`, + Example("Compare compression ratios across algorithms", `root.original_size = content().length() +root.gzip_size = content().compress("gzip").length() +root.lz4_size = content().compress("lz4").length()`, [2]string{ - `hello world I love space`, - `{"compressed":"BCJNGGRwuRgAAIBoZWxsbyB3b3JsZCBJIGxvdmUgc3BhY2UAAAAAGoETLg=="}`, + `The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog.`, + `{"gzip_size":114,"lz4_size":85,"original_size":89}`, }, ), func(args *bloblang.ParsedParams) (bloblang.Method, error) { @@ -50,20 +49,18 @@ root.b_len = $long_content.compress("gzip").length() bloblang.MustRegisterMethodV2("decompress", bloblang.NewPluginSpec(). Category(query.MethodCategoryEncoding). - Description(`Decompresses a string or byte array value according to a specified algorithm. The result of decompression `). - Param(bloblang.NewStringParam("algorithm").Description("One of `gzip`, `pgzip`, `zlib`, `bzip2`, `flate`, `snappy`, `lz4`, `zstd`.")). - Example("", `root = this.compressed.decode("base64").decompress("lz4")`, + Description(`Decompresses a byte array using the specified decompression algorithm. Returns decompressed data as bytes. Use with data that was previously compressed using the corresponding algorithm.`). + Param(bloblang.NewStringParam("algorithm").Description("The decompression algorithm: `gzip`, `pgzip` (parallel gzip), `zlib`, `bzip2`, `flate`, `snappy`, `lz4`, or `zstd`.")). + Example("Decompress base64-encoded compressed data", `root = this.compressed.decode("base64").decompress("gzip")`, [2]string{ - `{"compressed":"BCJNGGRwuRgAAIBoZWxsbyB3b3JsZCBJIGxvdmUgc3BhY2UAAAAAGoETLg=="}`, + `{"compressed":"H4sIAN12MWkAA8tIzcnJVyjPL8pJUfBUyMkvS1UoLkhMTgUAQpDxbxgAAAA="}`, `hello world I love space`, }, ). - Example( - "Use the `.string()` method in order to coerce the result into a string, this makes it possible to place the data within a JSON document without automatic base64 encoding.", - `root.result = this.compressed.decode("base64").decompress("lz4").string()`, + Example("Convert decompressed bytes to string for JSON output", `root.message = this.compressed.decode("base64").decompress("gzip").string()`, [2]string{ - `{"compressed":"BCJNGGRwuRgAAIBoZWxsbyB3b3JsZCBJIGxvdmUgc3BhY2UAAAAAGoETLg=="}`, - `{"result":"hello world I love space"}`, + `{"compressed":"H4sIAN12MWkAA8tIzcnJVyjPL8pJUfBUyMkvS1UoLkhMTgUAQpDxbxgAAAA="}`, + `{"message":"hello world I love space"}`, }, ), func(args *bloblang.ParsedParams) (bloblang.Method, error) { diff --git a/internal/impl/pure/bloblang_general.go b/internal/impl/pure/bloblang_general.go index 9a73b2266..3882df3fc 100644 --- a/internal/impl/pure/bloblang_general.go +++ b/internal/impl/pure/bloblang_general.go @@ -19,17 +19,17 @@ func init() { bloblang.NewPluginSpec(). Category(query.FunctionCategoryGeneral). Experimental(). - Description("Returns a non-negative integer that increments each time it is resolved, yielding the minimum (`1` by default) as the first value. Each instantiation of `counter` has its own independent count. Once the maximum integer (or `max` argument) is reached the counter resets back to the minimum."). + Description("Generates an incrementing sequence of integers starting from a minimum value (default 1). Each counter instance maintains its own independent state across message processing. When the maximum value is reached, the counter automatically resets to the minimum."). Param(bloblang.NewQueryParam("min", true). Default(1). - Description("The minimum value of the counter, this is the first value that will be yielded. If this parameter is dynamic it will be resolved only once during the lifetime of the mapping.")). + Description("The starting value of the counter. This is the first value yielded. Evaluated once when the mapping is initialized.")). Param(bloblang.NewQueryParam("max", true). Default(maxInt). - Description("The maximum value of the counter, once this value is yielded the counter will reset back to the min. If this parameter is dynamic it will be resolved only once during the lifetime of the mapping.")). + Description("The maximum value before the counter resets to min. Evaluated once when the mapping is initialized.")). Param(bloblang.NewQueryParam("set", false). Optional(). - Description("An optional mapping that when specified will be executed each time the counter is resolved. When this mapping resolves to a non-negative integer value it will cause the counter to reset to this value and yield it. If this mapping is omitted or doesn't resolve to anything then the counter will increment and yield the value as normal. If this mapping resolves to `null` then the counter is not incremented and the current value is yielded. If this mapping resolves to a deletion then the counter is reset to the `min` value.")). - Example("", `root.id = counter()`, + Description("An optional query that controls counter behavior: when it resolves to a non-negative integer, the counter is set to that value; when it resolves to `null`, the counter is read without incrementing; when it resolves to a deletion, the counter resets to min; otherwise the counter increments normally.")). + Example("Generate sequential IDs for each message.", `root.id = counter()`, [2]string{ `{}`, `{"id":1}`, @@ -39,51 +39,62 @@ func init() { `{"id":2}`, }, ). - Example("It's possible to increment a counter multiple times within a single mapping invocation using a map.", + Example("Use a custom range for the counter.", + `root.batch_num = counter(min: 100, max: 200)`, + [2]string{ + `{}`, + `{"batch_num":100}`, + }, + [2]string{ + `{}`, + `{"batch_num":101}`, + }, + ). + Example("Increment a counter multiple times within a single mapping using a named map.", ` -map foos { +map increment { root = counter() } -root.meow_id = null.apply("foos") -root.woof_id = null.apply("foos") +root.first_id = null.apply("increment") +root.second_id = null.apply("increment") `, [2]string{ `{}`, - `{"meow_id":1,"woof_id":2}`, + `{"first_id":1,"second_id":2}`, }, [2]string{ `{}`, - `{"meow_id":3,"woof_id":4}`, + `{"first_id":3,"second_id":4}`, }, ). Example( - "By specifying an optional `set` parameter it is possible to dynamically reset the counter based on input data.", - `root.consecutive_doggos = counter(min: 1, set: if !this.sound.lowercase().contains("woof") { 0 })`, + "Conditionally reset a counter based on input data.", + `root.streak = counter(set: if this.status != "success" { 0 })`, [2]string{ - `{"sound":"woof woof"}`, - `{"consecutive_doggos":1}`, + `{"status":"success"}`, + `{"streak":1}`, }, [2]string{ - `{"sound":"woofer wooooo"}`, - `{"consecutive_doggos":2}`, + `{"status":"success"}`, + `{"streak":2}`, }, [2]string{ - `{"sound":"meow"}`, - `{"consecutive_doggos":0}`, + `{"status":"failure"}`, + `{"streak":0}`, }, [2]string{ - `{"sound":"uuuuh uh uh woof uhhhhhh"}`, - `{"consecutive_doggos":1}`, + `{"status":"success"}`, + `{"streak":1}`, }, ). Example( - "The `set` parameter can also be utilized to peek at the counter without mutating it by returning `null`.", - `root.things = counter(set: if this.id == null { null })`, - [2]string{`{"id":"a"}`, `{"things":1}`}, - [2]string{`{"id":"b"}`, `{"things":2}`}, - [2]string{`{"what":"just checking"}`, `{"things":2}`}, - [2]string{`{"id":"c"}`, `{"things":3}`}, + "Peek at the current counter value without incrementing by using null in the set parameter.", + `root.count = counter(set: if this.peek { null })`, + [2]string{`{"peek":false}`, `{"count":1}`}, + [2]string{`{"peek":false}`, `{"count":2}`}, + [2]string{`{"peek":true}`, `{"count":2}`}, + [2]string{`{"peek":false}`, `{"count":3}`}, ), func(args *bloblang.ParsedParams) (bloblang.AdvancedFunction, error) { minFunc, err := args.GetQuery("min") diff --git a/internal/impl/pure/bloblang_time.go b/internal/impl/pure/bloblang_time.go index a8254de35..2aa47fac3 100644 --- a/internal/impl/pure/bloblang_time.go +++ b/internal/impl/pure/bloblang_time.go @@ -28,14 +28,20 @@ func init() { Beta(). Static(). Category(query.MethodCategoryTime). - Description(`Returns the result of rounding a timestamp to the nearest multiple of the argument duration (nanoseconds). The rounding behavior for halfway values is to round up. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The `+"<>"+` method can be used in order to parse different timestamp formats.`). + Description(`Rounds a timestamp to the nearest multiple of the specified duration. Halfway values round up. Accepts unix timestamps (seconds with optional decimal precision) or RFC 3339 formatted strings.`). Param(bloblang.NewInt64Param("duration").Description("A duration measured in nanoseconds to round by.")). Version("4.2.0"). - Example("Use the method `parse_duration` to convert a duration string into an integer argument.", + Example("Round timestamp to the nearest hour.", `root.created_at_hour = this.created_at.ts_round("1h".parse_duration())`, [2]string{ `{"created_at":"2020-08-14T05:54:23Z"}`, `{"created_at_hour":"2020-08-14T06:00:00Z"}`, + }). + Example("Round timestamp to the nearest minute.", + `root.created_at_minute = this.created_at.ts_round("1m".parse_duration())`, + [2]string{ + `{"created_at":"2020-08-14T05:54:23Z"}`, + `{"created_at_minute":"2020-08-14T05:54:00Z"}`, }) tsRoundCtor := func(args *bloblang.ParsedParams) (bloblang.Method, error) { @@ -55,14 +61,20 @@ func init() { Beta(). Static(). Category(query.MethodCategoryTime). - Description(`Returns the result of converting a timestamp to a specified timezone. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The `+"<>"+` method can be used in order to parse different timestamp formats.`). - Param(bloblang.NewStringParam("tz").Description(`The timezone to change to. If set to "UTC" then the timezone will be UTC. If set to "Local" then the local timezone will be used. Otherwise, the argument is taken to be a location name corresponding to a file in the IANA Time Zone database, such as "America/New_York".`)). + Description(`Converts a timestamp to a different timezone while preserving the moment in time. Accepts unix timestamps (seconds with optional decimal precision) or RFC 3339 formatted strings.`). + Param(bloblang.NewStringParam("tz").Description(`The timezone to change to. Use "UTC" for UTC, "Local" for local timezone, or an IANA Time Zone database location name like "America/New_York".`)). Version("4.3.0"). - Example("", + Example("Convert timestamp to UTC timezone.", `root.created_at_utc = this.created_at.ts_tz("UTC")`, [2]string{ `{"created_at":"2021-02-03T17:05:06+01:00"}`, `{"created_at_utc":"2021-02-03T16:05:06Z"}`, + }). + Example("Convert timestamp to a specific timezone.", + `root.created_at_ny = this.created_at.ts_tz("America/New_York")`, + [2]string{ + `{"created_at":"2021-02-03T16:05:06Z"}`, + `{"created_at_ny":"2021-02-03T11:05:06-05:00"}`, }) tsTZCtor := func(args *bloblang.ParsedParams) (bloblang.Method, error) { @@ -85,15 +97,39 @@ func init() { Category(query.MethodCategoryTime). Beta(). Static(). - Description("Parse parameter string as ISO 8601 period and add it to value with high precision for units larger than an hour."). - Param(bloblang.NewStringParam("duration").Description(`Duration in ISO 8601 format`)) + Description("Adds an ISO 8601 duration to a timestamp with calendar-aware precision for years, months, and days. Useful when you need to add durations that account for variable month lengths or leap years."). + Param(bloblang.NewStringParam("duration").Description(`Duration in ISO 8601 format (e.g., "P1Y2M3D" for 1 year, 2 months, 3 days)`)). + Example("Add one year to a timestamp.", + `root.next_year = this.created_at.ts_add_iso8601("P1Y")`, + [2]string{ + `{"created_at":"2020-08-14T05:54:23Z"}`, + `{"next_year":"2021-08-14T05:54:23Z"}`, + }). + Example("Add a complex duration with multiple units.", + `root.future_date = this.created_at.ts_add_iso8601("P1Y2M3DT4H5M6S")`, + [2]string{ + `{"created_at":"2020-01-01T00:00:00Z"}`, + `{"future_date":"2021-03-04T04:05:06Z"}`, + }) tsSubISOSpec := bloblang.NewPluginSpec(). Category(query.MethodCategoryTime). Beta(). Static(). - Description("Parse parameter string as ISO 8601 period and subtract it from value with high precision for units larger than an hour."). - Param(bloblang.NewStringParam("duration").Description(`Duration in ISO 8601 format`)) + Description("Subtracts an ISO 8601 duration from a timestamp with calendar-aware precision for years, months, and days. Useful when you need to subtract durations that account for variable month lengths or leap years."). + Param(bloblang.NewStringParam("duration").Description(`Duration in ISO 8601 format (e.g., "P1Y2M3D" for 1 year, 2 months, 3 days)`)). + Example("Subtract one year from a timestamp.", + `root.last_year = this.created_at.ts_sub_iso8601("P1Y")`, + [2]string{ + `{"created_at":"2020-08-14T05:54:23Z"}`, + `{"last_year":"2019-08-14T05:54:23Z"}`, + }). + Example("Subtract a complex duration with multiple units.", + `root.past_date = this.created_at.ts_sub_iso8601("P1Y2M3DT4H5M6S")`, + [2]string{ + `{"created_at":"2021-03-04T04:05:06Z"}`, + `{"past_date":"2020-01-01T00:00:00Z"}`, + }) tsModifyISOCtor := func(callback func(d period.Period, t time.Time) time.Time) func(args *bloblang.ParsedParams) (bloblang.Method, error) { return func(args *bloblang.ParsedParams) (bloblang.Method, error) { @@ -128,15 +164,15 @@ func init() { parseDurSpec := bloblang.NewPluginSpec(). Static(). Category(query.MethodCategoryTime). - Description(`Attempts to parse a string as a duration and returns an integer of nanoseconds. A duration string is a possibly signed sequence of decimal numbers, each with an optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".`). - Example("", + Description(`Parses a Go-style duration string into nanoseconds. A duration string is a signed sequence of decimal numbers with unit suffixes like "300ms", "-1.5h", or "2h45m". Valid units: "ns", "us" (or "µs"), "ms", "s", "m", "h".`). + Example("Parse microseconds to nanoseconds.", `root.delay_for_ns = this.delay_for.parse_duration()`, [2]string{ `{"delay_for":"50us"}`, `{"delay_for_ns":50000}`, }, ). - Example("", + Example("Parse hours to seconds.", `root.delay_for_s = this.delay_for.parse_duration() / 1000000000`, [2]string{ `{"delay_for":"2h"}`, @@ -160,27 +196,20 @@ func init() { Category(query.MethodCategoryTime). Beta(). Static(). - Description(`Attempts to parse a string using ISO-8601 rules as a duration and returns an integer of nanoseconds. A duration string is represented by the format "P[n]Y[n]M[n]DT[n]H[n]M[n]S" or "P[n]W". In these representations, the "[n]" is replaced by the value for each of the date and time elements that follow the "[n]". For example, "P3Y6M4DT12H30M5S" represents a duration of "three years, six months, four days, twelve hours, thirty minutes, and five seconds". The last field of the format allows fractions with one decimal place, so "P3.5S" will return 3500000000ns. Any additional decimals will be truncated.`). - Example("Arbitrary ISO-8601 duration string to nanoseconds:", + Description(`Parses an ISO 8601 duration string into nanoseconds. Format: "P[n]Y[n]M[n]DT[n]H[n]M[n]S" or "P[n]W". Example: "P3Y6M4DT12H30M5S" means 3 years, 6 months, 4 days, 12 hours, 30 minutes, 5 seconds. Supports fractional seconds with full precision (not just one decimal place).`). + Example("Parse complex ISO 8601 duration to nanoseconds.", `root.delay_for_ns = this.delay_for.parse_duration_iso8601()`, [2]string{ `{"delay_for":"P3Y6M4DT12H30M5S"}`, `{"delay_for_ns":110839937000000000}`, }, ). - Example("Two hours ISO-8601 duration string to seconds:", + Example("Parse hours to seconds.", `root.delay_for_s = this.delay_for.parse_duration_iso8601() / 1000000000`, [2]string{ `{"delay_for":"PT2H"}`, `{"delay_for_s":7200}`, }, - ). - Example("Two and a half seconds ISO-8601 duration string to seconds:", - `root.delay_for_s = this.delay_for.parse_duration_iso8601() / 1000000000`, - [2]string{ - `{"delay_for":"PT2.5S"}`, - `{"delay_for_s":2.5}`, - }, ) parseDurISOCtor := func(args *bloblang.ParsedParams) (bloblang.Method, error) { @@ -203,20 +232,25 @@ func init() { Category(query.MethodCategoryTime). Beta(). Static(). - Description(`Attempts to parse a string as a timestamp following a specified format and outputs a timestamp, which can then be fed into methods such as ` + "<>" + `. - -The input format is defined by showing how the reference time, defined to be Mon Jan 2 15:04:05 -0700 MST 2006, would be displayed if it were the value. For an alternative way to specify formats check out the ` + "<>" + ` method.`). - Param(bloblang.NewStringParam("format").Description("The format of the target string.")) + Description(`Parses a timestamp string using Go's reference time format and outputs a timestamp object. The format uses "Mon Jan 2 15:04:05 -0700 MST 2006" as a reference - show how this reference time would appear in your format. Use ts_strptime for strftime-style formats instead.`). + Param(bloblang.NewStringParam("format").Description("The format of the input string using Go's reference time.")) parseTSSpecDep := asDeprecated(parseTSSpec) parseTSSpec = parseTSSpec. - Example("", + Example("Parse a date with abbreviated month name.", `root.doc.timestamp = this.doc.timestamp.ts_parse("2006-Jan-02")`, [2]string{ `{"doc":{"timestamp":"2020-Aug-14"}}`, `{"doc":{"timestamp":"2020-08-14T00:00:00Z"}}`, }, + ). + Example("Parse a custom datetime format.", + `root.parsed = this.timestamp.ts_parse("Jan 2, 2006 at 3:04pm (MST)")`, + [2]string{ + `{"timestamp":"Aug 14, 2020 at 5:54am (UTC)"}`, + `{"parsed":"2020-08-14T05:54:00Z"}`, + }, ) parseTSCtor := func(deprecated bool) bloblang.MethodConstructorV2 { @@ -246,14 +280,14 @@ The input format is defined by showing how the reference time, defined to be Mon Category(query.MethodCategoryTime). Beta(). Static(). - Description("Attempts to parse a string as a timestamp following a specified strptime-compatible format and outputs a timestamp, which can then be fed into <>."). - Param(bloblang.NewStringParam("format").Description("The format of the target string.")) + Description("Parses a timestamp string using strptime format specifiers (like %Y, %m, %d) and outputs a timestamp object. Use ts_parse for Go-style reference time formats instead."). + Param(bloblang.NewStringParam("format").Description("The format string using strptime specifiers (e.g., %Y-%m-%d).")) parseTSStrptimeSpecDep := asDeprecated(parseTSStrptimeSpec) parseTSStrptimeSpec = parseTSStrptimeSpec. Example( - "The format consists of zero or more conversion specifiers and ordinary characters (except `%`). All ordinary characters are copied to the output string without modification. Each conversion specification begins with a `%` character followed by the character that determines the behavior of the specifier. Please refer to https://linux.die.net/man/3/strptime[man 3 strptime] for the list of format specifiers.", + "Parse date with abbreviated month using strptime format.", `root.doc.timestamp = this.doc.timestamp.ts_strptime("%Y-%b-%d")`, [2]string{ `{"doc":{"timestamp":"2020-Aug-14"}}`, @@ -261,7 +295,7 @@ The input format is defined by showing how the reference time, defined to be Mon }, ). Example( - "As an extension provided by the underlying formatting library, https://github.com/itchyny/timefmt-go[itchyny/timefmt-go], the `%f` directive is supported for zero-padded microseconds, which originates from Python. Note that E and O modifier characters are not supported.", + "Parse datetime with microseconds using %f directive.", `root.doc.timestamp = this.doc.timestamp.ts_strptime("%Y-%b-%d %H:%M:%S.%f")`, [2]string{ `{"doc":{"timestamp":"2020-Aug-14 11:50:26.371000"}}`, @@ -298,48 +332,25 @@ The input format is defined by showing how the reference time, defined to be Mon Category(query.MethodCategoryTime). Beta(). Static(). - Description(`Attempts to format a timestamp value as a string according to a specified format, or RFC 3339 by default. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. - -The output format is defined by showing how the reference time, defined to be Mon Jan 2 15:04:05 -0700 MST 2006, would be displayed if it were the value. For an alternative way to specify formats check out the ` + "<>" + ` method.`). - Param(bloblang.NewStringParam("format").Description("The output format to use.").Default(time.RFC3339Nano)). - Param(bloblang.NewStringParam("tz").Description("An optional timezone to use, otherwise the timezone of the input string is used, or in the case of unix timestamps the local timezone is used.").Optional()) + Description(`Formats a timestamp as a string using Go's reference time format. Defaults to RFC 3339 if no format specified. The format uses "Mon Jan 2 15:04:05 -0700 MST 2006" as a reference. Accepts unix timestamps (with decimal precision) or RFC 3339 strings. Use ts_strftime for strftime-style formats.`). + Param(bloblang.NewStringParam("format").Description("The output format using Go's reference time.").Default(time.RFC3339Nano)). + Param(bloblang.NewStringParam("tz").Description("Optional timezone (e.g., 'UTC', 'America/New_York'). Defaults to input timezone or local time for unix timestamps.").Optional()) formatTSSpecDep := asDeprecated(formatTSSpec) formatTSSpec = formatTSSpec. - Example("", - `root.something_at = (this.created_at + 300).ts_format()`, - // `{"created_at":1597405526}`, - // `{"something_at":"2020-08-14T11:50:26.371Z"}`, - ). - Example( - "An optional string argument can be used in order to specify the output format of the timestamp. The format is defined by showing how the reference time, defined to be Mon Jan 2 15:04:05 -0700 MST 2006, would be displayed if it were the value.", - `root.something_at = (this.created_at + 300).ts_format("2006-Jan-02 15:04:05")`, - // `{"created_at":1597405526}`, - // `{"something_at":"2020-Aug-14 11:50:26"}`, - ). - Example( - "A second optional string argument can also be used in order to specify a timezone, otherwise the timezone of the input string is used, or in the case of unix timestamps the local timezone is used.", - `root.something_at = this.created_at.ts_format(format: "2006-Jan-02 15:04:05", tz: "UTC")`, - [2]string{ - `{"created_at":1597405526}`, - `{"something_at":"2020-Aug-14 11:45:26"}`, - }, + Example("Format timestamp with custom format.", + `root.something_at = this.created_at.ts_format("2006-Jan-02 15:04:05")`, [2]string{ `{"created_at":"2020-08-14T11:50:26.371Z"}`, `{"something_at":"2020-Aug-14 11:50:26"}`, }, ). - Example( - "And `ts_format` supports up to nanosecond precision with floating point timestamp values.", - `root.something_at = this.created_at.ts_format("2006-Jan-02 15:04:05.999999", "UTC")`, - [2]string{ - `{"created_at":1597405526.123456}`, - `{"something_at":"2020-Aug-14 11:45:26.123456"}`, - }, + Example("Format unix timestamp with timezone specification.", + `root.something_at = this.created_at.ts_format(format: "2006-Jan-02 15:04:05", tz: "UTC")`, [2]string{ - `{"created_at":"2020-08-14T11:50:26.371Z"}`, - `{"something_at":"2020-Aug-14 11:50:26.371"}`, + `{"created_at":1597405526}`, + `{"something_at":"2020-Aug-14 11:45:26"}`, }, ) @@ -374,38 +385,24 @@ The output format is defined by showing how the reference time, defined to be Mo Category(query.MethodCategoryTime). Beta(). Static(). - Description("Attempts to format a timestamp value as a string according to a specified strftime-compatible format. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format."). - Param(bloblang.NewStringParam("format").Description("The output format to use.")). - Param(bloblang.NewStringParam("tz").Description("An optional timezone to use, otherwise the timezone of the input string is used.").Optional()) + Description("Formats a timestamp as a string using strptime format specifiers (like %Y, %m, %d). Accepts unix timestamps (with decimal precision) or RFC 3339 strings. Supports %f for microseconds. Use ts_format for Go-style reference time formats."). + Param(bloblang.NewStringParam("format").Description("The output format using strptime specifiers.")). + Param(bloblang.NewStringParam("tz").Description("Optional timezone. Defaults to input timezone or local time for unix timestamps.").Optional()) formatTSStrftimeSpecDep := asDeprecated(formatTSStrftimeSpec) formatTSStrftimeSpec = formatTSStrftimeSpec. Example( - "The format consists of zero or more conversion specifiers and ordinary characters (except `%`). All ordinary characters are copied to the output string without modification. Each conversion specification begins with `%` character followed by the character that determines the behavior of the specifier. Please refer to https://linux.die.net/man/3/strftime[man 3 strftime] for the list of format specifiers.", - `root.something_at = (this.created_at + 300).ts_strftime("%Y-%b-%d %H:%M:%S")`, - // `{"created_at":1597405526}`, - // `{"something_at":"2020-Aug-14 11:50:26"}`, - ). - Example( - "A second optional string argument can also be used in order to specify a timezone, otherwise the timezone of the input string is used, or in the case of unix timestamps the local timezone is used.", - `root.something_at = this.created_at.ts_strftime("%Y-%b-%d %H:%M:%S", "UTC")`, - [2]string{ - `{"created_at":1597405526}`, - `{"something_at":"2020-Aug-14 11:45:26"}`, - }, + "Format timestamp with strftime specifiers.", + `root.something_at = this.created_at.ts_strftime("%Y-%b-%d %H:%M:%S")`, [2]string{ `{"created_at":"2020-08-14T11:50:26.371Z"}`, `{"something_at":"2020-Aug-14 11:50:26"}`, }, ). Example( - "As an extension provided by the underlying formatting library, https://github.com/itchyny/timefmt-go[itchyny/timefmt-go], the `%f` directive is supported for zero-padded microseconds, which originates from Python. Note that E and O modifier characters are not supported.", + "Format with microseconds using %f directive.", `root.something_at = this.created_at.ts_strftime("%Y-%b-%d %H:%M:%S.%f", "UTC")`, - [2]string{ - `{"created_at":1597405526}`, - `{"something_at":"2020-Aug-14 11:45:26.000000"}`, - }, [2]string{ `{"created_at":"2020-08-14T11:50:26.371Z"}`, `{"something_at":"2020-Aug-14 11:50:26.371000"}`, @@ -443,17 +440,24 @@ The output format is defined by showing how the reference time, defined to be Mo Category(query.MethodCategoryTime). Beta(). Static(). - Description("Attempts to format a timestamp value as a unix timestamp. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The <> method can be used in order to parse different timestamp formats.") + Description("Converts a timestamp to a unix timestamp (seconds since epoch). Accepts unix timestamps or RFC 3339 strings. Returns an integer representing seconds.") formatTSUnixSpecDep := asDeprecated(formatTSUnixSpec) formatTSUnixSpec = formatTSUnixSpec. - Example("", + Example("Convert RFC 3339 timestamp to unix seconds.", `root.created_at_unix = this.created_at.ts_unix()`, [2]string{ `{"created_at":"2009-11-10T23:00:00Z"}`, `{"created_at_unix":1257894000}`, }, + ). + Example("Unix timestamp passthrough returns same value.", + `root.timestamp = this.ts.ts_unix()`, + [2]string{ + `{"ts":1257894000}`, + `{"timestamp":1257894000}`, + }, ) formatTSUnixCtor := func(args *bloblang.ParsedParams) (bloblang.Method, error) { @@ -470,17 +474,24 @@ The output format is defined by showing how the reference time, defined to be Mo Category(query.MethodCategoryTime). Beta(). Static(). - Description("Attempts to format a timestamp value as a unix timestamp with millisecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The <> method can be used in order to parse different timestamp formats.") + Description("Converts a timestamp to a unix timestamp with millisecond precision (milliseconds since epoch). Accepts unix timestamps or RFC 3339 strings. Returns an integer representing milliseconds.") formatTSUnixMilliSpecDep := asDeprecated(formatTSUnixMilliSpec) formatTSUnixMilliSpec = formatTSUnixMilliSpec. - Example("", + Example("Convert timestamp to milliseconds since epoch.", `root.created_at_unix = this.created_at.ts_unix_milli()`, [2]string{ `{"created_at":"2009-11-10T23:00:00Z"}`, `{"created_at_unix":1257894000000}`, }, + ). + Example("Useful for JavaScript timestamp compatibility.", + `root.js_timestamp = this.event_time.ts_unix_milli()`, + [2]string{ + `{"event_time":"2020-08-14T11:45:26.123Z"}`, + `{"js_timestamp":1597405526123}`, + }, ) formatTSUnixMilliCtor := func(args *bloblang.ParsedParams) (bloblang.Method, error) { @@ -497,17 +508,24 @@ The output format is defined by showing how the reference time, defined to be Mo Category(query.MethodCategoryTime). Beta(). Static(). - Description("Attempts to format a timestamp value as a unix timestamp with microsecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The <> method can be used in order to parse different timestamp formats.") + Description("Converts a timestamp to a unix timestamp with microsecond precision (microseconds since epoch). Accepts unix timestamps or RFC 3339 strings. Returns an integer representing microseconds.") formatTSUnixMicroSpecDep := asDeprecated(formatTSUnixMicroSpec) formatTSUnixMicroSpec = formatTSUnixMicroSpec. - Example("", + Example("Convert timestamp to microseconds since epoch.", `root.created_at_unix = this.created_at.ts_unix_micro()`, [2]string{ `{"created_at":"2009-11-10T23:00:00Z"}`, `{"created_at_unix":1257894000000000}`, }, + ). + Example("Preserve microsecond precision from timestamp.", + `root.precise_time = this.timestamp.ts_unix_micro()`, + [2]string{ + `{"timestamp":"2020-08-14T11:45:26.123456Z"}`, + `{"precise_time":1597405526123456}`, + }, ) formatTSUnixMicroCtor := func(args *bloblang.ParsedParams) (bloblang.Method, error) { @@ -524,17 +542,24 @@ The output format is defined by showing how the reference time, defined to be Mo Category(query.MethodCategoryTime). Beta(). Static(). - Description("Attempts to format a timestamp value as a unix timestamp with nanosecond precision. Timestamp values can either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in RFC 3339 format. The <> method can be used in order to parse different timestamp formats.") + Description("Converts a timestamp to a unix timestamp with nanosecond precision (nanoseconds since epoch). Accepts unix timestamps or RFC 3339 strings. Returns an integer representing nanoseconds.") formatTSUnixNanoSpecDep := asDeprecated(formatTSUnixNanoSpec) formatTSUnixNanoSpec = formatTSUnixNanoSpec. - Example("", + Example("Convert timestamp to nanoseconds since epoch.", `root.created_at_unix = this.created_at.ts_unix_nano()`, [2]string{ `{"created_at":"2009-11-10T23:00:00Z"}`, `{"created_at_unix":1257894000000000000}`, }, + ). + Example("Preserve full nanosecond precision.", + `root.precise_time = this.timestamp.ts_unix_nano()`, + [2]string{ + `{"timestamp":"2020-08-14T11:45:26.123456789Z"}`, + `{"precise_time":1597405526123456789}`, + }, ) formatTSUnixNanoCtor := func(args *bloblang.ParsedParams) (bloblang.Method, error) { @@ -551,14 +576,20 @@ The output format is defined by showing how the reference time, defined to be Mo Beta(). Static(). Category(query.MethodCategoryTime). - Description(`Returns the difference in nanoseconds between the target timestamp (t1) and the timestamp provided as a parameter (t2). The `+"<>"+` method can be used in order to parse different timestamp formats.`). - Param(bloblang.NewTimestampParam("t2").Description("The second timestamp to be subtracted from the method target.")). + Description(`Calculates the duration in nanoseconds between two timestamps (t1 - t2). Returns a signed integer: positive if t1 is after t2, negative if t1 is before t2. Use .abs() for absolute duration.`). + Param(bloblang.NewTimestampParam("t2").Description("The timestamp to subtract from the target timestamp.")). Version("4.23.0"). - Example("Use the `.abs()` method in order to calculate an absolute duration between two timestamps.", + Example("Calculate absolute duration between two timestamps.", `root.between = this.started_at.ts_sub("2020-08-14T05:54:23Z").abs()`, [2]string{ `{"started_at":"2020-08-13T05:54:23Z"}`, `{"between":86400000000000}`, + }). + Example("Calculate signed duration (can be negative).", + `root.duration_ns = this.end_time.ts_sub(this.start_time)`, + [2]string{ + `{"start_time":"2020-08-14T10:00:00Z","end_time":"2020-08-14T11:30:00Z"}`, + `{"duration_ns":5400000000000}`, }) tsSubCtor := func(args *bloblang.ParsedParams) (bloblang.Method, error) { From 65b14e41aec32c612df152348309e2c0c592c13b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Thu, 4 Dec 2025 11:22:06 +0100 Subject: [PATCH 38/51] bloblang: create category general and register functions without category under a proper category --- internal/bloblang/query/docs.go | 1 + internal/bloblang/query/methods.go | 36 ++++++++++--------- internal/bloblang/query/methods_strings.go | 12 +++++-- internal/bloblang/query/methods_structured.go | 5 ++- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/internal/bloblang/query/docs.go b/internal/bloblang/query/docs.go index caba6aeec..0db3b42b7 100644 --- a/internal/bloblang/query/docs.go +++ b/internal/bloblang/query/docs.go @@ -164,6 +164,7 @@ func NewHiddenFunctionSpec(name string) FunctionSpec { // Method categories. var ( + MethodCategoryGeneral = "General" MethodCategoryStrings = "String Manipulation" MethodCategoryNumbers = "Number Manipulation" MethodCategoryTime = "Timestamp Manipulation" diff --git a/internal/bloblang/query/methods.go b/internal/bloblang/query/methods.go index 40101f868..7a4983ac9 100644 --- a/internal/bloblang/query/methods.go +++ b/internal/bloblang/query/methods.go @@ -13,9 +13,8 @@ import ( ) var _ = registerMethod( - NewMethodSpec( - "apply", - "Apply a declared mapping to a target value.", + NewMethodSpec("apply", "Apply a declared mapping to a target value.").InCategory( + MethodCategoryGeneral, "", NewExampleSpec("", `map thing { root.inner = this.first @@ -146,9 +145,8 @@ func boolMethod(target Function, args *ParsedParams) (Function, error) { //------------------------------------------------------------------------------ var _ = registerMethod( - NewMethodSpec( - "catch", - "If the result of a target query fails (due to incorrect types, failed parsing, etc) the argument is returned instead.", + NewMethodSpec("catch", "If the result of a target query fails (due to incorrect types, failed parsing, etc) the argument is returned instead.").InCategory( + MethodCategoryGeneral, "", NewExampleSpec("", `root.doc.id = this.thing.id.string().catch(uuid_v4())`, ), @@ -185,9 +183,8 @@ func catchMethod(fn Function, args *ParsedParams) (Function, error) { //------------------------------------------------------------------------------ var _ = registerMethod( - NewMethodSpec( - "from", - "Modifies a target query such that certain functions are executed from the perspective of another message in the batch. This allows you to mutate events based on the contents of other messages. Functions that support this behavior are `content`, `json` and `meta`.", + NewMethodSpec("from", "Modifies a target query such that certain functions are executed from the perspective of another message in the batch. This allows you to mutate events based on the contents of other messages. Functions that support this behavior are `content`, `json` and `meta`.").InCategory( + MethodCategoryGeneral, "", NewExampleSpec("For example, the following map extracts the contents of the JSON field `foo` specifically from message index `1` of a batch, effectively overriding the field `foo` for all messages of a batch to that of message 1:", `root = this root.foo = json("foo").from(1)`, @@ -227,9 +224,8 @@ func (f *fromMethod) QueryTargets(ctx TargetsContext) (TargetsContext, []TargetP //------------------------------------------------------------------------------ var _ = registerMethod( - NewMethodSpec( - "from_all", - "Modifies a target query such that certain functions are executed from the perspective of each message in the batch, and returns the set of results as an array. Functions that support this behavior are `content`, `json` and `meta`.", + NewMethodSpec("from_all", "Modifies a target query such that certain functions are executed from the perspective of each message in the batch, and returns the set of results as an array. Functions that support this behavior are `content`, `json` and `meta`.").InCategory( + MethodCategoryGeneral, "", NewExampleSpec("", `root = this root.foo_summed = json("foo").from_all().sum()`, @@ -334,8 +330,9 @@ func getMethodCtor(target Function, args *ParsedParams) (Function, error) { //------------------------------------------------------------------------------ var _ = registerMethod( - NewHiddenMethodSpec("map"). - Param(ParamQuery("query", "A query to execute on the target.", false)), + NewMethodSpec("map", "Executes a query on the target value, allowing you to transform or extract data from the current context.").InCategory( + MethodCategoryGeneral, "", + ).Param(ParamQuery("query", "A query to execute on the target.", false)), mapMethod, ) @@ -366,7 +363,12 @@ func mapMethod(target Function, args *ParsedParams) (Function, error) { //------------------------------------------------------------------------------ -var _ = registerMethod(NewHiddenMethodSpec("not"), notMethodCtor) +var _ = registerMethod( + NewMethodSpec("not", "Returns the logical NOT (negation) of a boolean value. Converts true to false and false to true.").InCategory( + MethodCategoryGeneral, "", + ), + notMethodCtor, +) type notMethod struct { fn Function @@ -512,8 +514,8 @@ func timestampCoerceMethod(target Function, args *ParsedParams) (Function, error //------------------------------------------------------------------------------ var _ = registerMethod( - NewMethodSpec( - "or", "If the result of the target query fails or resolves to `null`, returns the argument instead. This is an explicit method alternative to the coalesce pipe operator `|`.", + NewMethodSpec("or", "If the result of the target query fails or resolves to `null`, returns the argument instead. This is an explicit method alternative to the coalesce pipe operator `|`.").InCategory( + MethodCategoryGeneral, "", NewExampleSpec("", `root.doc.id = this.thing.id.or(uuid_v4())`), ).Param(ParamQuery("fallback", "A value to yield, or query to execute, if the target query fails or resolves to `null`.", true)), orMethod, diff --git a/internal/bloblang/query/methods_strings.go b/internal/bloblang/query/methods_strings.go index 5fcdb86dc..82ba5e4f3 100644 --- a/internal/bloblang/query/methods_strings.go +++ b/internal/bloblang/query/methods_strings.go @@ -1570,7 +1570,9 @@ var _ = registerSimpleMethod( //------------------------------------------------------------------------------ var _ = registerSimpleMethod( - NewHiddenMethodSpec("replace"). + NewMethodSpec("replace", "Replaces all occurrences of a substring with another string. Use for text transformation, cleaning data, or normalizing strings.").InCategory( + MethodCategoryStrings, "", + ). Param(ParamString("old", "A string to match against.")). Param(ParamString("new", "A string to replace with.")), replaceAllImpl, @@ -1622,7 +1624,9 @@ func replaceAllImpl(args *ParsedParams) (simpleMethod, error) { //------------------------------------------------------------------------------ var _ = registerSimpleMethod( - NewHiddenMethodSpec("replace_many"). + NewMethodSpec("replace_many", "Performs multiple find-and-replace operations in sequence using an array of `[old, new]` pairs. More efficient than chaining multiple `replace_all` calls. Use for bulk text transformations.").InCategory( + MethodCategoryStrings, "", + ). Param(ParamArray("values", "An array of values, each even value will be replaced with the following odd value.")), replaceAllManyImpl, ) @@ -1966,7 +1970,9 @@ var _ = registerSimpleMethod( //------------------------------------------------------------------------------ var _ = registerSimpleMethod( - NewHiddenMethodSpec("re_replace"). + NewMethodSpec("re_replace", "Replaces all regex matches with a replacement string that can reference capture groups using `$1`, `$2`, etc. Use for pattern-based transformations or data reformatting.").InCategory( + MethodCategoryRegexp, "", + ). Param(ParamString("pattern", "The pattern to match against.")). Param(ParamString("value", "The value to replace with.")), reReplaceAllImpl, diff --git a/internal/bloblang/query/methods_structured.go b/internal/bloblang/query/methods_structured.go index 0c6a6f49c..e2ab55644 100644 --- a/internal/bloblang/query/methods_structured.go +++ b/internal/bloblang/query/methods_structured.go @@ -300,9 +300,8 @@ var _ = registerSimpleMethod( //------------------------------------------------------------------------------ var _ = registerSimpleMethod( - NewMethodSpec( - "exists", - "Checks whether a field exists at the specified dot path within an object. Returns true if the field is present (even if null), false otherwise.", + NewMethodSpec("exists", "Checks whether a field exists at the specified dot path within an object. Returns true if the field is present (even if null), false otherwise.").InCategory( + MethodCategoryObjectAndArray, "", NewExampleSpec("", `root.result = this.foo.exists("bar.baz")`, `{"foo":{"bar":{"baz":"yep, I exist"}}}`, From 8f599d383aaea5a83d4726774d55863ef17d0c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Thu, 4 Dec 2025 12:35:05 +0100 Subject: [PATCH 39/51] bloblang/query: InCategory aggregate examples into the top-level Examples field --- internal/bloblang/query/docs.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/bloblang/query/docs.go b/internal/bloblang/query/docs.go index 0db3b42b7..1f8d1061d 100644 --- a/internal/bloblang/query/docs.go +++ b/internal/bloblang/query/docs.go @@ -301,5 +301,9 @@ func (m MethodSpec) InCategory(category, description string, examples ...Example Examples: examples, }) m.Categories = cats + + // Aggregate examples into the top-level Examples field + m.Examples = append(m.Examples, examples...) + return m } From ccc4d1956bcc900118f2ffa81c085b2e0f282a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Thu, 4 Dec 2025 11:25:09 +0100 Subject: [PATCH 40/51] Update CL --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d33fd5c77..063ef6c71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,22 @@ Changelog All notable changes to this project will be documented in this file. +## 4.62.0 - TBD + +### Added + +- CLI: Add support for listing bloblang functions and methods with jsonschema. (@mmatczuk) +- CLI: Add input field to `blobl` command. (@mmatczuk) + +### Changed + +- Bloblang: Create category general and register functions without category under a proper category. (@mmatczuk) +- Bloblang: Function and method descriptions and examples overhaul. (@mmatczuk) + +### Fixed + +- CLI: Fix data race in `blobl` command where program exits before printing output. (@mmatczuk) + ## 4.61.0 - 2025-11-21 ### Added From 1ea1057251cb2d478abbbb5f04b254176c30553b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 08:38:24 +0000 Subject: [PATCH 41/51] build(deps): bump the production-dependencies group across 1 directory with 15 updates Bumps the production-dependencies group with 10 updates in the / directory: | Package | From | To | | --- | --- | --- | | cuelang.org/go | `0.13.2` | `0.15.1` | | [github.com/golang-jwt/jwt/v5](https://github.com/golang-jwt/jwt) | `5.2.2` | `5.3.0` | | [github.com/itchyny/gojq](https://github.com/itchyny/gojq) | `0.12.17` | `0.12.18` | | [github.com/klauspost/compress](https://github.com/klauspost/compress) | `1.18.0` | `1.18.2` | | [github.com/linkedin/goavro/v2](https://github.com/linkedin/goavro) | `2.14.0` | `2.14.1` | | [github.com/rickb777/period](https://github.com/rickb777/period) | `1.0.15` | `1.0.21` | | [github.com/stretchr/testify](https://github.com/stretchr/testify) | `1.10.0` | `1.11.1` | | [go.opentelemetry.io/otel](https://github.com/open-telemetry/opentelemetry-go) | `1.37.0` | `1.38.0` | | [golang.org/x/crypto](https://github.com/golang/crypto) | `0.43.0` | `0.45.0` | | [github.com/gofrs/uuid/v5](https://github.com/gofrs/uuid) | `5.3.2` | `5.4.0` | Updates `cuelang.org/go` from 0.13.2 to 0.15.1 Updates `github.com/golang-jwt/jwt/v5` from 5.2.2 to 5.3.0 - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Commits](https://github.com/golang-jwt/jwt/compare/v5.2.2...v5.3.0) Updates `github.com/itchyny/gojq` from 0.12.17 to 0.12.18 - [Release notes](https://github.com/itchyny/gojq/releases) - [Changelog](https://github.com/itchyny/gojq/blob/main/CHANGELOG.md) - [Commits](https://github.com/itchyny/gojq/compare/v0.12.17...v0.12.18) Updates `github.com/itchyny/timefmt-go` from 0.1.6 to 0.1.7 - [Release notes](https://github.com/itchyny/timefmt-go/releases) - [Changelog](https://github.com/itchyny/timefmt-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/itchyny/timefmt-go/compare/v0.1.6...v0.1.7) Updates `github.com/klauspost/compress` from 1.18.0 to 1.18.2 - [Release notes](https://github.com/klauspost/compress/releases) - [Commits](https://github.com/klauspost/compress/compare/v1.18.0...v1.18.2) Updates `github.com/linkedin/goavro/v2` from 2.14.0 to 2.14.1 - [Release notes](https://github.com/linkedin/goavro/releases) - [Commits](https://github.com/linkedin/goavro/compare/v2.14.0...v2.14.1) Updates `github.com/rickb777/period` from 1.0.15 to 1.0.21 - [Release notes](https://github.com/rickb777/period/releases) - [Commits](https://github.com/rickb777/period/compare/v1.0.15...v1.0.21) Updates `github.com/stretchr/testify` from 1.10.0 to 1.11.1 - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.10.0...v1.11.1) Updates `go.opentelemetry.io/otel` from 1.37.0 to 1.38.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.37.0...v1.38.0) Updates `go.opentelemetry.io/otel/trace` from 1.37.0 to 1.38.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.37.0...v1.38.0) Updates `golang.org/x/crypto` from 0.43.0 to 0.45.0 - [Commits](https://github.com/golang/crypto/compare/v0.43.0...v0.45.0) Updates `golang.org/x/oauth2` from 0.30.0 to 0.32.0 - [Commits](https://github.com/golang/oauth2/compare/v0.30.0...v0.32.0) Updates `golang.org/x/sync` from 0.17.0 to 0.18.0 - [Commits](https://github.com/golang/sync/compare/v0.17.0...v0.18.0) Updates `golang.org/x/text` from 0.30.0 to 0.31.0 - [Release notes](https://github.com/golang/text/releases) - [Commits](https://github.com/golang/text/compare/v0.30.0...v0.31.0) Updates `github.com/gofrs/uuid/v5` from 5.3.2 to 5.4.0 - [Release notes](https://github.com/gofrs/uuid/releases) - [Commits](https://github.com/gofrs/uuid/compare/v5.3.2...v5.4.0) --- updated-dependencies: - dependency-name: cuelang.org/go dependency-version: 0.15.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: github.com/golang-jwt/jwt/v5 dependency-version: 5.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: github.com/itchyny/gojq dependency-version: 0.12.18 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: github.com/itchyny/timefmt-go dependency-version: 0.1.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: github.com/klauspost/compress dependency-version: 1.18.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: github.com/linkedin/goavro/v2 dependency-version: 2.14.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: github.com/rickb777/period dependency-version: 1.0.21 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: github.com/stretchr/testify dependency-version: 1.11.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: go.opentelemetry.io/otel dependency-version: 1.38.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: go.opentelemetry.io/otel/trace dependency-version: 1.38.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: golang.org/x/crypto dependency-version: 0.45.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: golang.org/x/oauth2 dependency-version: 0.32.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: golang.org/x/sync dependency-version: 0.18.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: golang.org/x/text dependency-version: 0.31.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: github.com/gofrs/uuid/v5 dependency-version: 5.4.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 36 ++++++++++---------- go.sum | 104 ++++++++++++++++++++++++++++++--------------------------- 2 files changed, 72 insertions(+), 68 deletions(-) diff --git a/go.mod b/go.mod index 844bdfc1b..e8e68f6ec 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/redpanda-data/benthos/v4 require ( - cuelang.org/go v0.13.2 + cuelang.org/go v0.15.1 github.com/Jeffail/gabs/v2 v2.7.0 github.com/Jeffail/grok v1.1.0 github.com/Jeffail/shutdown v1.1.0 @@ -10,40 +10,40 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.18.0 github.com/fsnotify/fsnotify v1.9.0 - github.com/golang-jwt/jwt/v5 v5.2.2 + github.com/golang-jwt/jwt/v5 v5.3.0 github.com/gorilla/handlers v1.5.2 github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.3 github.com/hashicorp/golang-lru/arc/v2 v2.0.7 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/influxdata/go-syslog/v3 v3.0.0 - github.com/itchyny/gojq v0.12.17 - github.com/itchyny/timefmt-go v0.1.6 + github.com/itchyny/gojq v0.12.18 + github.com/itchyny/timefmt-go v0.1.7 github.com/jmespath/go-jmespath v0.4.0 - github.com/klauspost/compress v1.18.0 + github.com/klauspost/compress v1.18.2 github.com/klauspost/pgzip v1.2.6 - github.com/linkedin/goavro/v2 v2.14.0 + github.com/linkedin/goavro/v2 v2.14.1 github.com/matoous/go-nanoid/v2 v2.1.0 github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 github.com/pierrec/lz4/v4 v4.1.22 github.com/quipo/dependencysolver v0.0.0-20170801134659-2b009cb4ddcc github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 - github.com/rickb777/period v1.0.15 + github.com/rickb777/period v1.0.21 github.com/robfig/cron/v3 v3.0.1 github.com/segmentio/ksuid v1.0.4 github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/tilinna/z85 v1.0.0 github.com/urfave/cli/v2 v2.27.7 github.com/xeipuuv/gojsonschema v1.2.0 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a - go.opentelemetry.io/otel v1.37.0 - go.opentelemetry.io/otel/trace v1.37.0 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/trace v1.38.0 go.uber.org/multierr v1.11.0 - golang.org/x/crypto v0.43.0 - golang.org/x/oauth2 v0.30.0 - golang.org/x/sync v0.17.0 - golang.org/x/text v0.30.0 + golang.org/x/crypto v0.45.0 + golang.org/x/oauth2 v0.32.0 + golang.org/x/sync v0.18.0 + golang.org/x/text v0.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -55,21 +55,21 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/gofrs/uuid/v5 v5.3.2 + github.com/gofrs/uuid/v5 v5.4.0 github.com/golang/snappy v0.0.4 // indirect github.com/govalues/decimal v0.1.36 // indirect github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rickb777/plural v1.4.4 // indirect + github.com/rickb777/plural v1.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel/metric v1.37.0 // indirect - golang.org/x/sys v0.37.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + golang.org/x/sys v0.38.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 49b4223f3..7281f76f9 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ -cuelabs.dev/go/oci/ociregistry v0.0.0-20250304105642-27e071d2c9b1 h1:Dmbd5Q+ENb2C6carvwrMsrOUwJ9X9qfL5JdW32gYAHo= -cuelabs.dev/go/oci/ociregistry v0.0.0-20250304105642-27e071d2c9b1/go.mod h1:dqrnoZx62xbOZr11giMPrWbhlaV8euHwciXZEy3baT8= -cuelang.org/go v0.13.2 h1:SagzeEASX4E2FQnRbItsqa33sSelrJjQByLqH9uZCE8= -cuelang.org/go v0.13.2/go.mod h1:8MoQXu+RcXsa2s9mebJN1HJ1orVDc9aI9/yKi6Dzsi4= +cuelabs.dev/go/oci/ociregistry v0.0.0-20250722084951-074d06050084 h1:4k1yAtPvZJZQTu8DRY8muBo0LHv6TqtrE0AO5n6IPYs= +cuelabs.dev/go/oci/ociregistry v0.0.0-20250722084951-074d06050084/go.mod h1:4WWeZNxUO1vRoZWAHIG0KZOd6dA25ypyWuwD3ti0Tdc= +cuelang.org/go v0.15.1 h1:MRnjc/KJE+K42rnJ3a+425f1jqXeOOgq9SK4tYRTtWw= +cuelang.org/go v0.15.1/go.mod h1:NYw6n4akZcTjA7QQwJ1/gqWrrhsN4aZwhcAL0jv9rZE= github.com/Jeffail/gabs/v2 v2.7.0 h1:Y2edYaTcE8ZpRsR2AtmPu5xQdFDIthFG0jYhu5PY8kg= github.com/Jeffail/gabs/v2 v2.7.0/go.mod h1:dp5ocw1FvBBQYssgHsG7I1WYsiLRtkUaB1FEtSwvNUw= github.com/Jeffail/grok v1.1.0 h1:kiHmZ+0J5w/XUihRgU3DY9WIxKrNQCDjnfAb6bMLFaE= @@ -21,8 +21,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/emicklei/proto v1.14.0 h1:WYxC0OrBuuC+FUCTZvb8+fzEHdZMwLEF+OnVfZA3LXU= -github.com/emicklei/proto v1.14.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= +github.com/emicklei/proto v1.14.2 h1:wJPxPy2Xifja9cEMrcA/g08art5+7CGJNFNk35iXC1I= +github.com/emicklei/proto v1.14.2/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -36,10 +36,10 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= -github.com/gofrs/uuid/v5 v5.3.2 h1:2jfO8j3XgSwlz/wHqemAEugfnTlikAYHhnqQ8Xh4fE0= -github.com/gofrs/uuid/v5 v5.3.2/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= -github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= -github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/gofrs/uuid/v5 v5.4.0 h1:EfbpCTjqMuGyq5ZJwxqzn3Cbr2d0rUZU7v5ycAk/e/0= +github.com/gofrs/uuid/v5 v5.4.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -62,16 +62,16 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/influxdata/go-syslog/v3 v3.0.0 h1:jichmjSZlYK0VMmlz+k4WeOQd7z745YLsvGMqwtYt4I= github.com/influxdata/go-syslog/v3 v3.0.0/go.mod h1:tulsOp+CecTAYC27u9miMgq21GqXRW6VdKbOG+QSP4Q= -github.com/itchyny/gojq v0.12.17 h1:8av8eGduDb5+rvEdaOO+zQUjA04MS0m3Ps8HiD+fceg= -github.com/itchyny/gojq v0.12.17/go.mod h1:WBrEMkgAfAGO1LUcGOckBl5O726KPp+OlkKug0I/FEY= -github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q= -github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg= +github.com/itchyny/gojq v0.12.18 h1:gFGHyt/MLbG9n6dqnvlliiya2TaMMh6FFaR2b1H6Drc= +github.com/itchyny/gojq v0.12.18/go.mod h1:4hPoZ/3lN9fDL1D+aK7DY1f39XZpY9+1Xpjz8atrEkg= +github.com/itchyny/timefmt-go v0.1.7 h1:xyftit9Tbw+Dc/huSSPJaEmX1TVL8lw5vxjJLK4GMMA= +github.com/itchyny/timefmt-go v0.1.7/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -81,8 +81,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/linkedin/goavro/v2 v2.14.0 h1:aNO/js65U+Mwq4yB5f1h01c3wiM458qtRad1DN0CMUI= -github.com/linkedin/goavro/v2 v2.14.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk= +github.com/linkedin/goavro/v2 v2.14.1 h1:/8VjDpd38PRsy02JS0jflAu7JZPfJcGTwqWgMkFS2iI= +github.com/linkedin/goavro/v2 v2.14.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk= github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -104,18 +104,18 @@ github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/protocolbuffers/txtpbfmt v0.0.0-20250129171521-feedd8250727 h1:A8EM8fVuYc0qbVMw9D6EiKdKTIm1SmLvAWcCc2mipGY= -github.com/protocolbuffers/txtpbfmt v0.0.0-20250129171521-feedd8250727/go.mod h1:VmWrOlMnBZNtToCWzRlZlIXcJqjo0hS5dwQbRD62gL8= +github.com/protocolbuffers/txtpbfmt v0.0.0-20251016062345-16587c79cd91 h1:s1LvMaU6mVwoFtbxv/rCZKE7/fwDmDY684FfUe4c1Io= +github.com/protocolbuffers/txtpbfmt v0.0.0-20251016062345-16587c79cd91/go.mod h1:JSbkp0BviKovYYt9XunS95M3mLPibE9bGg+Y95DsEEY= github.com/quipo/dependencysolver v0.0.0-20170801134659-2b009cb4ddcc h1:hK577yxEJ2f5s8w2iy2KimZmgrdAUZUNftE1ESmg2/Q= github.com/quipo/dependencysolver v0.0.0-20170801134659-2b009cb4ddcc/go.mod h1:OQt6Zo5B3Zs+C49xul8kcHo+fZ1mCLPvd0LFxiZ2DHc= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rickb777/expect v0.24.0 h1:IzFxn4jINkVuCmx4jdQP7LxaIBhG60bDVbeGWk3xnzo= -github.com/rickb777/expect v0.24.0/go.mod h1:jwwS3gmukQ7wPxzEtOhMJEv43UxSwOBE7MUgTt8CX0k= -github.com/rickb777/period v1.0.15 h1:nWR4rgCtImT0CXw5kAsjHv+ExCEFt/18zAySOi7pWI8= -github.com/rickb777/period v1.0.15/go.mod h1:3lWluyeZEk6n1jfLCPG4dH3C0N3NxjmYL4Dmcxip3es= -github.com/rickb777/plural v1.4.4 h1:OpZU8uRr9P2NkYAbkLMwlKNVJyJ5HvRcRBFyXGJtKGI= -github.com/rickb777/plural v1.4.4/go.mod h1:DB19dtrplGS5s6VJVHn7tvmFYPoE83p1xqio3oVnNRM= +github.com/rickb777/expect v1.0.6 h1:1JE3CfYGyuhN5OTu5nqvIF3VjS2AoqoctcGlbNTVl3w= +github.com/rickb777/expect v1.0.6/go.mod h1:raunaduUM/p8CzpTZeDmoexwlIFF+Peg0Mj/p//9mkA= +github.com/rickb777/period v1.0.21 h1:6Cn7+Nv00dmxcThwQi3Kl+HqsYe0dPs6PgwAE/TX73o= +github.com/rickb777/period v1.0.21/go.mod h1:liTmui1MSVgOqkJemF3K6c35CqiEHp0oGHCNZIXnIMA= +github.com/rickb777/plural v1.4.7 h1:rBRAxp9aTFYzWTLWIE/UTwKcaqSSAV2ml7aOUFYpAGo= +github.com/rickb777/plural v1.4.7/go.mod h1:DB19dtrplGS5s6VJVHn7tvmFYPoE83p1xqio3oVnNRM= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -133,8 +133,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tilinna/z85 v1.0.0 h1:uqFnJBlD01dosSeo5sK1G1YGbPuwqVHqR+12OJDRjUw= github.com/tilinna/z85 v1.0.0/go.mod h1:EfpFU/DUY4ddEy6CRvk2l+UQNEzHbh+bqBQS+04Nkxs= github.com/trivago/grok v1.0.0 h1:oV2ljyZT63tgXkmgEHg2U0jMqiKKuL0hkn49s6aRavQ= @@ -156,39 +156,43 @@ github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqTosly github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= -golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= -golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= -golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM= -golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= +golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= -golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= -golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= -golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 6fd6c25e4276d56890b14df828c16f9f35aba764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Fri, 5 Dec 2025 10:00:26 +0100 Subject: [PATCH 42/51] service: fixe race condition in TestForceTimelyNacksBatchedNoAck The test's Eventually check was passing immediately on array length instead of waiting for the 5ms timeout to fire. This caused the test to call ackFn before the automatic nack, canceling the timeout and setting the ack result to nil instead of the expected error. Update the check to verify the specific timeout error message, ensuring the automatic nack completes before proceeding. --- public/service/input_force_timely_nacks_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/public/service/input_force_timely_nacks_test.go b/public/service/input_force_timely_nacks_test.go index 982bf2379..0d8d62a20 100644 --- a/public/service/input_force_timely_nacks_test.go +++ b/public/service/input_force_timely_nacks_test.go @@ -190,9 +190,14 @@ func TestForceTimelyNacksBatchedNoAck(t *testing.T) { require.Eventually(t, func() bool { readerImpl.ackRcvdMut.Lock() - ackLen := len(readerImpl.ackRcvd) - readerImpl.ackRcvdMut.Unlock() - return ackLen >= 1 + defer readerImpl.ackRcvdMut.Unlock() + if len(readerImpl.ackRcvd) < 1 { + return false + } + // Wait for the timeout to actually fire and update the error + // The initial value is errors.New("ack not received"), but after + // timeout fires it should be errForceTimelyNacks + return readerImpl.ackRcvd[0] != nil && readerImpl.ackRcvd[0].Error() == "message acknowledgement exceeded maximum wait duration and has been rejected" }, time.Second, time.Millisecond*10) readerImpl.ackRcvdMut.Lock() From 0c669e008e4069c3e5a13b02772dbd43dd2b71b4 Mon Sep 17 00:00:00 2001 From: Tyler Rockwood Date: Mon, 8 Dec 2025 10:01:34 -0600 Subject: [PATCH 43/51] otel: update tracer name (#331) --- internal/tracing/otel.go | 2 +- internal/tracing/v2/otel.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/tracing/otel.go b/internal/tracing/otel.go index 29c1ce4e0..865ec6b29 100644 --- a/internal/tracing/otel.go +++ b/internal/tracing/otel.go @@ -14,7 +14,7 @@ import ( ) const ( - name = "benthos" + name = "redpanda-connect" ) // GetSpan returns a span attached to a message part. Returns nil if the part diff --git a/internal/tracing/v2/otel.go b/internal/tracing/v2/otel.go index 8eb970f9b..e5bf4b2ec 100644 --- a/internal/tracing/v2/otel.go +++ b/internal/tracing/v2/otel.go @@ -14,7 +14,7 @@ import ( ) const ( - name = "benthos" + name = "redpanda-connect" ) // GetSpan returns a span attached to a message part. Returns nil if the part From 15225b0dbd10f3502fead8a80f897b21f25333c9 Mon Sep 17 00:00:00 2001 From: Tyler Rockwood Date: Mon, 8 Dec 2025 10:06:27 -0600 Subject: [PATCH 44/51] otel: missed one instance of benthos (#332) --- public/service/config_extract_tracing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/service/config_extract_tracing.go b/public/service/config_extract_tracing.go index d83094c6c..e2ac4b990 100644 --- a/public/service/config_extract_tracing.go +++ b/public/service/config_extract_tracing.go @@ -115,7 +115,7 @@ func (s *spanInjectBatchInput) ReadBatch(ctx context.Context) (MessageBatch, Ack textProp := otel.GetTextMapPropagator() for i, p := range m { ctx := textProp.Extract(p.Context(), c) - pCtx, _ := prov.Tracer("benthos").Start(ctx, operationName) + pCtx, _ := prov.Tracer("redpanda-connect").Start(ctx, operationName) m[i] = p.WithContext(pCtx) } return m, afn, nil From c8eba2d9887b8d3c22e504f88111ad0f0cf8b09b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Mon, 8 Dec 2025 17:25:50 +0100 Subject: [PATCH 45/51] bloblang/query: add description to map_each --- internal/bloblang/query/methods_structured.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/bloblang/query/methods_structured.go b/internal/bloblang/query/methods_structured.go index e2ab55644..f49e3eca3 100644 --- a/internal/bloblang/query/methods_structured.go +++ b/internal/bloblang/query/methods_structured.go @@ -969,7 +969,7 @@ var _ = registerSimpleMethod( NewMethodSpec( "map_each", "", ).InCategory( - MethodCategoryObjectAndArray, "", + MethodCategoryObjectAndArray, "Applies a mapping query to each element of an array or each value in an object. Returns a new collection with the transformed values.", NewExampleSpec(`##### On arrays Transforms each array element using a query. Return deleted() to remove an element, or the new value to replace it.`, From ab5e020a7e93e1025cfce643f7dd1089444cfee2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Dec 2025 12:10:34 +0000 Subject: [PATCH 46/51] build(deps): bump the production-dependencies group with 2 updates Bumps the production-dependencies group with 2 updates: [golang.org/x/oauth2](https://github.com/golang/oauth2) and [golang.org/x/sync](https://github.com/golang/sync). Updates `golang.org/x/oauth2` from 0.32.0 to 0.34.0 - [Commits](https://github.com/golang/oauth2/compare/v0.32.0...v0.34.0) Updates `golang.org/x/sync` from 0.18.0 to 0.19.0 - [Commits](https://github.com/golang/sync/compare/v0.18.0...v0.19.0) --- updated-dependencies: - dependency-name: golang.org/x/oauth2 dependency-version: 0.34.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: golang.org/x/sync dependency-version: 0.19.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index e8e68f6ec..067669396 100644 --- a/go.mod +++ b/go.mod @@ -41,8 +41,8 @@ require ( go.opentelemetry.io/otel/trace v1.38.0 go.uber.org/multierr v1.11.0 golang.org/x/crypto v0.45.0 - golang.org/x/oauth2 v0.32.0 - golang.org/x/sync v0.18.0 + golang.org/x/oauth2 v0.34.0 + golang.org/x/sync v0.19.0 golang.org/x/text v0.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 7281f76f9..b4464034f 100644 --- a/go.sum +++ b/go.sum @@ -175,10 +175,10 @@ golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= -golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From c6802ec8dcebea8e57e9fb40b58419f7dc432bab Mon Sep 17 00:00:00 2001 From: Artem Klevtsov Date: Tue, 9 Dec 2025 01:16:01 +0700 Subject: [PATCH 47/51] feat: add missing_key field --- internal/impl/pure/processor_template.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/impl/pure/processor_template.go b/internal/impl/pure/processor_template.go index 89965292f..78667db89 100644 --- a/internal/impl/pure/processor_template.go +++ b/internal/impl/pure/processor_template.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "errors" + "fmt" "text/template" "github.com/redpanda-data/benthos/v4/internal/bundle" @@ -76,6 +77,10 @@ input: service.NewBloblangField("mapping"). Description("An optional xref:guides:bloblang/about.adoc[Bloblang] mapping to apply to the message before executing the template. This allows you to transform the data structure before templating."). Optional(), + service.NewStringEnumField("missing_key", "default", "invalid", "zero", "error"). + Description("Control the behavior during execution if a map is indexed with a key that is not present in the map."). + Default("default"). + Optional(), ) } @@ -106,11 +111,18 @@ func templateFromParsed(conf *service.ParsedConfig, mgr bundle.NewManagement) (* return nil, err } + option, err := conf.FieldString("missing_key") + if err != nil { + return nil, err + } + + option = fmt.Sprintf("missingkey=%s", option) + if code == "" && len(files) == 0 { return nil, errors.New("at least one of 'code' or 'files' fields must be specified") } - t := &tmplProc{tmpl: template.New("root")} + t := &tmplProc{tmpl: template.New("root").Option(option)} if len(files) > 0 { for _, f := range files { if t.tmpl, err = t.tmpl.ParseGlob(f); err != nil { From 0104c23b6da538cf96ac1fb54ab5d2d612089d32 Mon Sep 17 00:00:00 2001 From: Artem Klevtsov Date: Tue, 9 Dec 2025 01:16:14 +0700 Subject: [PATCH 48/51] feeat: add template tests --- internal/impl/pure/processor_template_test.go | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 internal/impl/pure/processor_template_test.go diff --git a/internal/impl/pure/processor_template_test.go b/internal/impl/pure/processor_template_test.go new file mode 100644 index 000000000..a2e93fdc8 --- /dev/null +++ b/internal/impl/pure/processor_template_test.go @@ -0,0 +1,84 @@ +package pure + +import ( + "testing" + + "github.com/redpanda-data/benthos/v4/internal/component/testutil" + "github.com/redpanda-data/benthos/v4/internal/manager/mock" + "github.com/redpanda-data/benthos/v4/internal/message" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTemplate(t *testing.T) { + conf, err := testutil.ProcessorFromYAML(` +template: + code: "{{ .name }}" +`) + require.NoError(t, err) + + tmpl, err := mock.NewManager().NewProcessor(conf) + require.NoError(t, err) + + msgIn := message.QuickBatch([][]byte{[]byte(`{"name": "John Doe"}`)}) + msgsOut, err := tmpl.ProcessBatch(t.Context(), msgIn) + require.NoError(t, err) + require.Len(t, msgsOut, 1) + require.Len(t, msgsOut[0], 1) + assert.Equal(t, "John Doe", string(msgsOut[0][0].AsBytes())) + + type testCase struct { + name string + input []string + expected []string + } + + tests := []testCase{ + { + name: "template test 1", + input: []string{`{"name": "John Doe"}`}, + expected: []string{`John Doe`}, + }, + { + name: "template test 2", + input: []string{`{"wrong": "John Doe"}`}, + expected: []string{``}, + }, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + msg := message.QuickBatch(nil) + for _, s := range test.input { + msg = append(msg, message.NewPart([]byte(s))) + } + msgs, res := tmpl.ProcessBatch(t.Context(), msg) + require.NoError(t, res) + + resStrs := []string{} + for _, b := range message.GetAllBytes(msgs[0]) { + resStrs = append(resStrs, string(b)) + } + assert.Equal(t, test.expected, resStrs) + }) + } +} + +func TestTemplateError(t *testing.T) { + conf, err := testutil.ProcessorFromYAML(` +template: + missing_key: 'error' + code: '{{ .name }}' +`) + require.NoError(t, err) + + tmpl, err := mock.NewManager().NewProcessor(conf) + require.NoError(t, err) + + msgIn := message.QuickBatch([][]byte{[]byte(`{"wrong": "John Doe"}`)}) + msgsOut, err := tmpl.ProcessBatch(t.Context(), msgIn) + require.NoError(t, err) + require.Len(t, msgsOut, 1) + require.Len(t, msgsOut[0], 1) + assert.Equal(t, string(msgIn[0].AsBytes()), string(msgsOut[0][0].AsBytes())) +} From 96537eb0aa1bbb68ba23d54731f8018933073070 Mon Sep 17 00:00:00 2001 From: Artem Klevtsov Date: Tue, 9 Dec 2025 01:21:12 +0700 Subject: [PATCH 49/51] fix" linter error --- internal/impl/pure/processor_template.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/impl/pure/processor_template.go b/internal/impl/pure/processor_template.go index 78667db89..4e77f3a71 100644 --- a/internal/impl/pure/processor_template.go +++ b/internal/impl/pure/processor_template.go @@ -6,7 +6,6 @@ import ( "bytes" "context" "errors" - "fmt" "text/template" "github.com/redpanda-data/benthos/v4/internal/bundle" @@ -116,7 +115,7 @@ func templateFromParsed(conf *service.ParsedConfig, mgr bundle.NewManagement) (* return nil, err } - option = fmt.Sprintf("missingkey=%s", option) + option = "missingkey=" + option if code == "" && len(files) == 0 { return nil, errors.New("at least one of 'code' or 'files' fields must be specified") From ad3433996e75f0e805b7aad34f2e8ad14c2e7d69 Mon Sep 17 00:00:00 2001 From: Artem Klevtsov Date: Tue, 9 Dec 2025 01:25:30 +0700 Subject: [PATCH 50/51] fix: format imports --- internal/impl/pure/processor_template_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/impl/pure/processor_template_test.go b/internal/impl/pure/processor_template_test.go index a2e93fdc8..e9b0949ed 100644 --- a/internal/impl/pure/processor_template_test.go +++ b/internal/impl/pure/processor_template_test.go @@ -3,11 +3,12 @@ package pure import ( "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/redpanda-data/benthos/v4/internal/component/testutil" "github.com/redpanda-data/benthos/v4/internal/manager/mock" "github.com/redpanda-data/benthos/v4/internal/message" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestTemplate(t *testing.T) { From 094f1e1dfc58816aa9224555b96d3272f23407c7 Mon Sep 17 00:00:00 2001 From: Artem Klevtsov Date: Tue, 9 Dec 2025 10:42:18 +0700 Subject: [PATCH 51/51] feaT: add array example --- internal/impl/pure/processor_template.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/internal/impl/pure/processor_template.go b/internal/impl/pure/processor_template.go index 4e77f3a71..4c324344f 100644 --- a/internal/impl/pure/processor_template.go +++ b/internal/impl/pure/processor_template.go @@ -59,11 +59,28 @@ input: generate: count: 1 mapping: root.foo = "bar" + +pipeline: processors: - template: code: | {{ template "greeting" . }} files: ["./templates/greeting.tmpl"] +`). + Example( + "Execute template on array", + `This example uses a xref:components:inputs/generate.adoc[`+"`generate`"+` input] to make payload for the template.`, + ` +input: + generate: + count: 1 + mapping: root = [1, 2, 3] + +pipeline: + processors: + - template: + missing_key: error + code: "{{ range . }}{{ . }}\n{{ end }}" `). Fields( service.NewStringField("code").