From a0aaa38af4f36d3aaa6f414bcf1040059d5b4940 Mon Sep 17 00:00:00 2001 From: asaharn Date: Thu, 17 Apr 2025 16:45:10 +0530 Subject: [PATCH 01/28] Added MS Fabric output plugin --- plugins/outputs/all/microsoft_fabric.go | 5 + .../outputs/microsoft_fabric/event_house.go | 94 +++++++++++ .../outputs/microsoft_fabric/event_hubs.go | 154 ++++++++++++++++++ .../outputs/microsoft_fabric/fabric_output.go | 10 ++ .../microsoft_fabric/microsoft_fabric.go | 113 +++++++++++++ plugins/outputs/microsoft_fabric/sample.conf | 61 +++++++ 6 files changed, 437 insertions(+) create mode 100644 plugins/outputs/all/microsoft_fabric.go create mode 100644 plugins/outputs/microsoft_fabric/event_house.go create mode 100644 plugins/outputs/microsoft_fabric/event_hubs.go create mode 100644 plugins/outputs/microsoft_fabric/fabric_output.go create mode 100644 plugins/outputs/microsoft_fabric/microsoft_fabric.go create mode 100644 plugins/outputs/microsoft_fabric/sample.conf diff --git a/plugins/outputs/all/microsoft_fabric.go b/plugins/outputs/all/microsoft_fabric.go new file mode 100644 index 0000000000000..d8e1602289739 --- /dev/null +++ b/plugins/outputs/all/microsoft_fabric.go @@ -0,0 +1,5 @@ +//go:build !custom || outputs || outputs.microsoft_fabric + +package all + +import _ "github.com/influxdata/telegraf/plugins/outputs/microsoft_fabric" // register plugin diff --git a/plugins/outputs/microsoft_fabric/event_house.go b/plugins/outputs/microsoft_fabric/event_house.go new file mode 100644 index 0000000000000..997c49faac699 --- /dev/null +++ b/plugins/outputs/microsoft_fabric/event_house.go @@ -0,0 +1,94 @@ +package microsoft_fabric + +import ( + "fmt" + "time" + + "github.com/Azure/azure-kusto-go/kusto/ingest" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + adx "github.com/influxdata/telegraf/plugins/common/adx" + "github.com/influxdata/telegraf/plugins/serializers/json" +) + +type EventHouse struct { + Config *adx.Config `toml:"cluster_config"` + client *adx.Client + log telegraf.Logger + serializer telegraf.Serializer +} + +func (e *EventHouse) Init() error { + serializer := &json.Serializer{ + TimestampUnits: config.Duration(time.Nanosecond), + TimestampFormat: time.RFC3339Nano, + } + if err := serializer.Init(); err != nil { + return err + } + e.serializer = serializer + return nil +} + +func (e *EventHouse) Connect() error { + var err error + if e.client, err = e.Config.NewClient("Kusto.Telegraf", e.log); err != nil { + return fmt.Errorf("creating new client failed: %w", err) + } + return nil +} + +func (e *EventHouse) Write(metrics []telegraf.Metric) error { + if e.Config.MetricsGrouping == adx.TablePerMetric { + return e.writeTablePerMetric(metrics) + } + return e.writeSingleTable(metrics) +} + +func (e *EventHouse) Close() error { + return e.client.Close() +} + +func (e *EventHouse) writeTablePerMetric(metrics []telegraf.Metric) error { + tableMetricGroups := make(map[string][]byte) + // Group metrics by name and serialize them + for _, m := range metrics { + tableName := m.Name() + metricInBytes, err := e.serializer.Serialize(m) + if err != nil { + return err + } + if existingBytes, ok := tableMetricGroups[tableName]; ok { + tableMetricGroups[tableName] = append(existingBytes, metricInBytes...) + } else { + tableMetricGroups[tableName] = metricInBytes + } + } + + // Push the metrics for each table + format := ingest.FileFormat(ingest.JSON) + for tableName, tableMetrics := range tableMetricGroups { + if err := e.client.PushMetrics(format, tableName, tableMetrics); err != nil { + return err + } + } + + return nil +} + +func (e *EventHouse) writeSingleTable(metrics []telegraf.Metric) error { + // serialise each metric in metrics - store in byte[] + metricsArray := make([]byte, 0) + for _, m := range metrics { + metricsInBytes, err := e.serializer.Serialize(m) + if err != nil { + return err + } + metricsArray = append(metricsArray, metricsInBytes...) + } + + // push metrics to a single table + format := ingest.FileFormat(ingest.JSON) + err := e.client.PushMetrics(format, e.Config.TableName, metricsArray) + return err +} diff --git a/plugins/outputs/microsoft_fabric/event_hubs.go b/plugins/outputs/microsoft_fabric/event_hubs.go new file mode 100644 index 0000000000000..d913eee6655d3 --- /dev/null +++ b/plugins/outputs/microsoft_fabric/event_hubs.go @@ -0,0 +1,154 @@ +//go:generate ../../../tools/readme_config_includer/generator +package microsoft_fabric + +import ( + "context" + _ "embed" + "errors" + "fmt" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/serializers/json" +) + +type EventHubs struct { + PartitionKey string `toml:"partition_key"` + MaxMessageSize config.Size `toml:"max_message_size"` + Timeout config.Duration `toml:"timeout"` + + connectionString string + log telegraf.Logger + client *azeventhubs.ProducerClient + options azeventhubs.EventDataBatchOptions + serializer telegraf.Serializer +} + +func (e *EventHubs) Init() error { + e.serializer = &json.Serializer{ + TimestampUnits: config.Duration(time.Nanosecond), + TimestampFormat: time.RFC3339Nano, + } + if e.MaxMessageSize > 0 { + e.options.MaxBytes = uint64(e.MaxMessageSize) + } + + return nil +} + +func (e *EventHubs) Connect() error { + cfg := &azeventhubs.ProducerClientOptions{ + ApplicationID: internal.FormatFullVersion(), + RetryOptions: azeventhubs.RetryOptions{MaxRetries: -1}, + } + + client, err := azeventhubs.NewProducerClientFromConnectionString(e.connectionString, "", cfg) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + e.client = client + + return nil +} + +func (e *EventHubs) Close() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(e.Timeout)) + defer cancel() + + return e.client.Close(ctx) +} + +func (e *EventHubs) SetSerializer(serializer telegraf.Serializer) { + e.serializer = serializer +} + +func (e *EventHubs) Write(metrics []telegraf.Metric) error { + ctx := context.Background() + + batchOptions := e.options + batches := make(map[string]*azeventhubs.EventDataBatch) + for i := 0; i < len(metrics); i++ { + m := metrics[i] + + // Prepare the payload + payload, err := e.serializer.Serialize(m) + if err != nil { + e.log.Errorf("Could not serialize metric: %v", err) + e.log.Tracef("metric: %+v", m) + continue + } + + // Get the batcher for the chosen partition + partition := "" + batchOptions.PartitionKey = nil + if e.PartitionKey != "" { + if key, ok := m.GetTag(e.PartitionKey); ok { + partition = key + batchOptions.PartitionKey = &partition + } else if key, ok := m.GetField(e.PartitionKey); ok { + if k, ok := key.(string); ok { + partition = k + batchOptions.PartitionKey = &partition + } + } + } + if _, found := batches[partition]; !found { + batches[partition], err = e.client.NewEventDataBatch(ctx, &batchOptions) + if err != nil { + return fmt.Errorf("creating batch for partition %q failed: %w", partition, err) + } + } + + // Add the event to the partition and send it if the batch is full + err = batches[partition].AddEventData(&azeventhubs.EventData{Body: payload}, nil) + if err == nil { + continue + } + + // If the event doesn't fit into the batch anymore, send the batch + if !errors.Is(err, azeventhubs.ErrEventDataTooLarge) { + return fmt.Errorf("adding metric to batch for partition %q failed: %w", partition, err) + } + + // The event is larger than the maximum allowed size so there + // is nothing we can do here but have to drop the metric. + if batches[partition].NumEvents() == 0 { + e.log.Errorf("Metric with %d bytes exceeds the maximum allowed size and must be dropped!", len(payload)) + e.log.Tracef("metric: %+v", m) + continue + } + if err := e.send(batches[partition]); err != nil { + return fmt.Errorf("sending batch for partition %q failed: %w", partition, err) + } + + // Create a new metric and reiterate over the current metric to be + // added in the next iteration of the for loop. + batches[partition], err = e.client.NewEventDataBatch(ctx, &e.options) + if err != nil { + return fmt.Errorf("creating batch for partition %q failed: %w", partition, err) + } + i-- + } + + // Send the remaining batches that never exceeded the batch size + for partition, batch := range batches { + if batch.NumBytes() == 0 { + continue + } + if err := e.send(batch); err != nil { + return fmt.Errorf("sending batch for partition %q failed: %w", partition, err) + } + } + return nil +} + +func (e *EventHubs) send(batch *azeventhubs.EventDataBatch) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(e.Timeout)) + defer cancel() + + return e.client.SendEventDataBatch(ctx, batch, nil) +} diff --git a/plugins/outputs/microsoft_fabric/fabric_output.go b/plugins/outputs/microsoft_fabric/fabric_output.go new file mode 100644 index 0000000000000..d32daf3f0d865 --- /dev/null +++ b/plugins/outputs/microsoft_fabric/fabric_output.go @@ -0,0 +1,10 @@ +package microsoft_fabric + +import "github.com/influxdata/telegraf" + +type FabricOutput interface { + Init() error + Connect() error + Write(metrics []telegraf.Metric) error + Close() error +} diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric.go b/plugins/outputs/microsoft_fabric/microsoft_fabric.go new file mode 100644 index 0000000000000..9de0d1d4d80cf --- /dev/null +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric.go @@ -0,0 +1,113 @@ +//go:generate ../../../tools/readme_config_includer/generator +package microsoft_fabric + +import ( + _ "embed" + "errors" + "strings" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + adx "github.com/influxdata/telegraf/plugins/common/adx" + "github.com/influxdata/telegraf/plugins/outputs" +) + +//go:embed sample.conf +var sampleConfig string + +type MicrosoftFabric struct { + ConnectionString string `toml:"connection_string"` + Log telegraf.Logger `toml:"-"` + Eventhouse *EventHouse `toml:"eventhouse_conf"` + Eventhubs *EventHubs `toml:"eventhubs_conf"` + activePlugin FabricOutput +} + +// Close implements telegraf.Output. +func (m *MicrosoftFabric) Close() error { + if m.activePlugin == nil { + return errors.New("no active plugin to close") + } + return m.activePlugin.Close() +} + +// Connect implements telegraf.Output. +func (m *MicrosoftFabric) Connect() error { + if m.activePlugin == nil { + return errors.New("no active plugin to connect") + } + return m.activePlugin.Connect() +} + +// SampleConfig implements telegraf.Output. +func (m *MicrosoftFabric) SampleConfig() string { + return sampleConfig +} + +// Write implements telegraf.Output. +func (m *MicrosoftFabric) Write(metrics []telegraf.Metric) error { + if m.activePlugin == nil { + return errors.New("no active plugin to write to") + } + return m.activePlugin.Write(metrics) +} + +func (m *MicrosoftFabric) Init() error { + ConnectionString := m.ConnectionString + + if ConnectionString == "" { + return errors.New("endpoint must not be empty. For Kusto refer : https://learn.microsoft.com/kusto/api/connection-strings/kusto?view=microsoft-fabric for EventHouse refer : https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/add-manage-eventstream-sources?pivots=enhanced-capabilities") + } + + if strings.HasPrefix(ConnectionString, "Endpoint=sb") { + m.Log.Info("Detected EventHouse endpoint, using EventHouse output plugin") + m.Eventhubs.connectionString = ConnectionString + m.Eventhubs.log = m.Log + m.Eventhubs.Init() + m.activePlugin = m.Eventhubs + } else if isKustoEndpoint(strings.ToLower(ConnectionString)) { + m.Log.Info("Detected Kusto endpoint, using Kusto output plugin") + //Setting up the AzureDataExplorer plugin initial properties + m.Eventhouse.Config.Endpoint = ConnectionString + m.Eventhouse.log = m.Log + m.Eventhouse.Init() + m.activePlugin = m.Eventhouse + } else { + return errors.New("invalid connection string. For Kusto refer : https://learn.microsoft.com/kusto/api/connection-strings/kusto?view=microsoft-fabric for EventHouse refer : https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/add-manage-eventstream-sources?pivots=enhanced-capabilities") + } + return nil +} + +func isKustoEndpoint(endpoint string) bool { + prefixes := []string{ + "data source=", + "addr=", + "address=", + "network address=", + "server=", + } + + for _, prefix := range prefixes { + if strings.HasPrefix(endpoint, prefix) { + return true + } + } + return false +} + +func init() { + + outputs.Add("microsoft_fabric", func() telegraf.Output { + return &MicrosoftFabric{ + Eventhubs: &EventHubs{ + Timeout: config.Duration(30 * time.Second), + }, + Eventhouse: &EventHouse{ + Config: &adx.Config{ + Timeout: config.Duration(30 * time.Second), + }, + }, + } + }) +} diff --git a/plugins/outputs/microsoft_fabric/sample.conf b/plugins/outputs/microsoft_fabric/sample.conf new file mode 100644 index 0000000000000..1531eb174c9ba --- /dev/null +++ b/plugins/outputs/microsoft_fabric/sample.conf @@ -0,0 +1,61 @@ +# Sends metrics to Microsoft Fabric +[[outputs.microsoft_fabric]] + ## The URI property of the Eventhouse resource on Azure + ## ex: connection_string = "Data Source=https://myadxresource.australiasoutheast.kusto.windows.net" + connection_string = "" + + + [outputs.microsoft_fabric.eventhouse_conf] + [outputs.microsoft_fabric.eventhouse_conf.cluster_config] + ## The Eventhouse database that the metrics will be ingested into. + ## The plugin will NOT generate this database automatically, it's expected that this database already exists before ingestion. + ## ex: "exampledatabase" + database = "" + + ## Timeout for Eventhouse operations + # timeout = "20s" + + ## Type of metrics grouping used when pushing to Eventhouse. + ## Default is "TablePerMetric" for one table per different metric. + ## For more information, please check the plugin README. + # metrics_grouping_type = "TablePerMetric" + + ## Name of the single table to store all the metrics (Only needed if metrics_grouping_type is "SingleTable"). + # table_name = "" + + ## Creates tables and relevant mapping if set to true(default). + ## Skips table and mapping creation if set to false, this is useful for running Telegraf with the lowest possible permissions i.e. table ingestor role. + # create_tables = true + + ## Ingestion method to use. + ## Available options are + ## - managed -- streaming ingestion with fallback to batched ingestion or the "queued" method below + ## - queued -- queue up metrics data and process sequentially + # ingestion_type = "queued" + + [outputs.microsoft_fabric.eventhubs_conf] + ## The full connection string to the Event Hub (required) + ## The shared access key must have "Send" permissions on the target Event Hub. + + ## Client timeout (defaults to 30s) + # timeout = "30s" + + ## Partition key + ## Metric tag or field name to use for the event partition key. The value of + ## this tag or field is set as the key for events if it exists. If both, tag + ## and field, exist the tag is preferred. + # partition_key = "" + + ## Set the maximum batch message size in bytes + ## The allowable size depends on the Event Hub tier + ## See: https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas#basic-vs-standard-vs-premium-vs-dedicated-tiers + ## Setting this to 0 means using the default size from the Azure Event Hubs Client library (1000000 bytes) + # max_message_size = 1000000 + + ## Data format to output. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md + data_format = "json" + + \ No newline at end of file From 868358f15462a3f8d7d4f28c19a1f5796fa2dad9 Mon Sep 17 00:00:00 2001 From: asaharn Date: Thu, 17 Apr 2025 17:06:19 +0530 Subject: [PATCH 02/28] Added Tests Renamed Eventhubs --- .../{event_hubs.go => event_stream.go} | 14 +- .../microsoft_fabric/microsoft_fabric.go | 45 +++-- .../microsoft_fabric/microsoft_fabric_test.go | 183 ++++++++++++++++++ plugins/outputs/microsoft_fabric/sample.conf | 2 +- 4 files changed, 220 insertions(+), 24 deletions(-) rename plugins/outputs/microsoft_fabric/{event_hubs.go => event_stream.go} (92%) create mode 100644 plugins/outputs/microsoft_fabric/microsoft_fabric_test.go diff --git a/plugins/outputs/microsoft_fabric/event_hubs.go b/plugins/outputs/microsoft_fabric/event_stream.go similarity index 92% rename from plugins/outputs/microsoft_fabric/event_hubs.go rename to plugins/outputs/microsoft_fabric/event_stream.go index d913eee6655d3..179c6f3b371cb 100644 --- a/plugins/outputs/microsoft_fabric/event_hubs.go +++ b/plugins/outputs/microsoft_fabric/event_stream.go @@ -16,7 +16,7 @@ import ( "github.com/influxdata/telegraf/plugins/serializers/json" ) -type EventHubs struct { +type EventStream struct { PartitionKey string `toml:"partition_key"` MaxMessageSize config.Size `toml:"max_message_size"` Timeout config.Duration `toml:"timeout"` @@ -28,7 +28,7 @@ type EventHubs struct { serializer telegraf.Serializer } -func (e *EventHubs) Init() error { +func (e *EventStream) Init() error { e.serializer = &json.Serializer{ TimestampUnits: config.Duration(time.Nanosecond), TimestampFormat: time.RFC3339Nano, @@ -40,7 +40,7 @@ func (e *EventHubs) Init() error { return nil } -func (e *EventHubs) Connect() error { +func (e *EventStream) Connect() error { cfg := &azeventhubs.ProducerClientOptions{ ApplicationID: internal.FormatFullVersion(), RetryOptions: azeventhubs.RetryOptions{MaxRetries: -1}, @@ -55,18 +55,18 @@ func (e *EventHubs) Connect() error { return nil } -func (e *EventHubs) Close() error { +func (e *EventStream) Close() error { ctx, cancel := context.WithTimeout(context.Background(), time.Duration(e.Timeout)) defer cancel() return e.client.Close(ctx) } -func (e *EventHubs) SetSerializer(serializer telegraf.Serializer) { +func (e *EventStream) SetSerializer(serializer telegraf.Serializer) { e.serializer = serializer } -func (e *EventHubs) Write(metrics []telegraf.Metric) error { +func (e *EventStream) Write(metrics []telegraf.Metric) error { ctx := context.Background() batchOptions := e.options @@ -146,7 +146,7 @@ func (e *EventHubs) Write(metrics []telegraf.Metric) error { return nil } -func (e *EventHubs) send(batch *azeventhubs.EventDataBatch) error { +func (e *EventStream) send(batch *azeventhubs.EventDataBatch) error { ctx, cancel := context.WithTimeout(context.Background(), time.Duration(e.Timeout)) defer cancel() diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric.go b/plugins/outputs/microsoft_fabric/microsoft_fabric.go index 9de0d1d4d80cf..d47e3427e151b 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric.go @@ -20,7 +20,7 @@ type MicrosoftFabric struct { ConnectionString string `toml:"connection_string"` Log telegraf.Logger `toml:"-"` Eventhouse *EventHouse `toml:"eventhouse_conf"` - Eventhubs *EventHubs `toml:"eventhubs_conf"` + Eventstream *EventStream `toml:"eventstream_conf"` activePlugin FabricOutput } @@ -54,27 +54,40 @@ func (m *MicrosoftFabric) Write(metrics []telegraf.Metric) error { } func (m *MicrosoftFabric) Init() error { - ConnectionString := m.ConnectionString + connectionString := m.ConnectionString - if ConnectionString == "" { - return errors.New("endpoint must not be empty. For Kusto refer : https://learn.microsoft.com/kusto/api/connection-strings/kusto?view=microsoft-fabric for EventHouse refer : https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/add-manage-eventstream-sources?pivots=enhanced-capabilities") + if connectionString == "" { + return errors.New("endpoint must not be empty. For EventHouse refer : " + + "https://learn.microsoft.com/kusto/api/connection-strings/kusto?view=microsoft-fabric " + + "for EventStream refer : https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/add-manage-eventstream-sources" + + "?pivots=enhanced-capabilities") } - if strings.HasPrefix(ConnectionString, "Endpoint=sb") { + if strings.HasPrefix(connectionString, "Endpoint=sb") { + m.Log.Info("Detected EventStream endpoint, using EventStream output plugin") + + m.Eventstream.connectionString = connectionString + m.Eventstream.log = m.Log + err := m.Eventstream.Init() + if err != nil { + return errors.New("error initializing EventStream plugin: " + err.Error()) + } + m.activePlugin = m.Eventstream + } else if isKustoEndpoint(strings.ToLower(connectionString)) { m.Log.Info("Detected EventHouse endpoint, using EventHouse output plugin") - m.Eventhubs.connectionString = ConnectionString - m.Eventhubs.log = m.Log - m.Eventhubs.Init() - m.activePlugin = m.Eventhubs - } else if isKustoEndpoint(strings.ToLower(ConnectionString)) { - m.Log.Info("Detected Kusto endpoint, using Kusto output plugin") - //Setting up the AzureDataExplorer plugin initial properties - m.Eventhouse.Config.Endpoint = ConnectionString + // Setting up the AzureDataExplorer plugin initial properties + m.Eventhouse.Config.Endpoint = connectionString m.Eventhouse.log = m.Log - m.Eventhouse.Init() + err := m.Eventhouse.Init() + if err != nil { + return errors.New("error initializing EventHouse plugin: " + err.Error()) + } m.activePlugin = m.Eventhouse } else { - return errors.New("invalid connection string. For Kusto refer : https://learn.microsoft.com/kusto/api/connection-strings/kusto?view=microsoft-fabric for EventHouse refer : https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/add-manage-eventstream-sources?pivots=enhanced-capabilities") + return errors.New("invalid connection string. For EventHouse refer : " + + "https://learn.microsoft.com/kusto/api/connection-strings/kusto?view=microsoft-fabric" + + " for EventStream refer : https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/" + + "add-manage-eventstream-sources?pivots=enhanced-capabilities") } return nil } @@ -100,7 +113,7 @@ func init() { outputs.Add("microsoft_fabric", func() telegraf.Output { return &MicrosoftFabric{ - Eventhubs: &EventHubs{ + Eventstream: &EventStream{ Timeout: config.Duration(30 * time.Second), }, Eventhouse: &EventHouse{ diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go new file mode 100644 index 0000000000000..66ba99e212509 --- /dev/null +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go @@ -0,0 +1,183 @@ +package microsoft_fabric + +import ( + "testing" + "time" + + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/plugins/common/adx" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +type MockOutput struct { + mock.Mock +} + +func (m *MockOutput) Connect() error { + args := m.Called() + return args.Error(0) +} + +func (m *MockOutput) Close() error { + args := m.Called() + return args.Error(0) +} + +func (m *MockOutput) Write(metrics []telegraf.Metric) error { + args := m.Called(metrics) + return args.Error(0) +} + +func (m *MockOutput) Init() error { + args := m.Called() + return args.Error(0) +} + +func TestMicrosoftFabric_Connect(t *testing.T) { + mockOutput := new(MockOutput) + mockOutput.On("Connect").Return(nil) + + plugin := MicrosoftFabric{ + activePlugin: mockOutput, + } + + err := plugin.Connect() + require.NoError(t, err) + mockOutput.AssertExpectations(t) +} + +func TestMicrosoftFabric_Close(t *testing.T) { + mockOutput := new(MockOutput) + mockOutput.On("Close").Return(nil) + + plugin := MicrosoftFabric{ + activePlugin: mockOutput, + } + + err := plugin.Close() + require.NoError(t, err) + mockOutput.AssertExpectations(t) +} + +func TestMicrosoftFabric_Write(t *testing.T) { + mockOutput := new(MockOutput) + mockOutput.On("Write", mock.Anything).Return(nil) + + plugin := MicrosoftFabric{ + activePlugin: mockOutput, + } + + metrics := []telegraf.Metric{ + testutil.TestMetric(1.0, "test_metric"), + } + + err := plugin.Write(metrics) + require.NoError(t, err) + mockOutput.AssertExpectations(t) +} + +func TestIsKustoEndpoint(t *testing.T) { + testCases := []struct { + name string + endpoint string + expected bool + }{ + { + name: "Valid address prefix", + endpoint: "address=https://example.com", + expected: true, + }, + { + name: "Valid network address prefix", + endpoint: "network address=https://example.com", + expected: true, + }, + { + name: "Valid server prefix", + endpoint: "server=https://example.com", + expected: true, + }, + { + name: "Invalid prefix", + endpoint: "https://example.com", + expected: false, + }, + { + name: "Empty endpoint", + endpoint: "", + expected: false, + }, + } + + for _, tC := range testCases { + t.Run(tC.name, func(t *testing.T) { + result := isKustoEndpoint(tC.endpoint) + require.Equal(t, tC.expected, result) + }) + } +} + +func TestMicrosoftFabric_Init(t *testing.T) { + tests := []struct { + name string + connectionString string + expectedError string + }{ + { + name: "Empty connection string", + connectionString: "", + expectedError: "endpoint must not be empty. For EventHouse refer : " + + "https://learn.microsoft.com/kusto/api/connection-strings/kusto?view=microsoft-fabric" + + " for EventStream refer : " + + "https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/add-manage-eventstream-sources?pivots=enhanced-capabilities", + }, + { + name: "Invalid connection string", + connectionString: "invalid_connection_string", + expectedError: "invalid connection string. For EventHouse refer : " + + "https://learn.microsoft.com/kusto/api/connection-strings/kusto?view=microsoft-fabric" + + " for EventStream refer : " + + "https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/add-manage-eventstream-sources?pivots=enhanced-capabilities", + }, + { + name: "Valid EventHouse connection string", + connectionString: "Endpoint=sb://namespace.servicebus.windows.net/;" + + "SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=superSecret1234;EntityPath=hubName", + expectedError: "", + }, + { + name: "Valid Kusto connection string", + connectionString: "data source=https://example.kusto.windows.net;Database=e2e", + expectedError: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mf := &MicrosoftFabric{ + ConnectionString: tt.connectionString, + Log: testutil.Logger{}, + Eventhouse: &EventHouse{ + Config: &adx.Config{ + Database: "database", + }, + }, + Eventstream: &EventStream{ + Timeout: config.Duration(30 * time.Second), + }, + } + err := mf.Init() + if tt.expectedError != "" { + require.Error(t, err) + assert.Equal(t, tt.expectedError, err.Error()) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/plugins/outputs/microsoft_fabric/sample.conf b/plugins/outputs/microsoft_fabric/sample.conf index 1531eb174c9ba..f49b45a0d7cbf 100644 --- a/plugins/outputs/microsoft_fabric/sample.conf +++ b/plugins/outputs/microsoft_fabric/sample.conf @@ -33,7 +33,7 @@ ## - queued -- queue up metrics data and process sequentially # ingestion_type = "queued" - [outputs.microsoft_fabric.eventhubs_conf] + [outputs.microsoft_fabric.eventstream_conf] ## The full connection string to the Event Hub (required) ## The shared access key must have "Send" permissions on the target Event Hub. From fdf2fb100bd440d396ed2348dcef474d2e404a4d Mon Sep 17 00:00:00 2001 From: asaharn Date: Thu, 17 Apr 2025 23:05:27 +0530 Subject: [PATCH 03/28] test cases added Lint fixes --- .../outputs/microsoft_fabric/event_house.go | 3 +- .../outputs/microsoft_fabric/event_stream.go | 7 ++-- .../microsoft_fabric/microsoft_fabric.go | 5 ++- .../microsoft_fabric/microsoft_fabric_test.go | 33 ++++++++++++++++--- 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/event_house.go b/plugins/outputs/microsoft_fabric/event_house.go index 997c49faac699..d29431083da94 100644 --- a/plugins/outputs/microsoft_fabric/event_house.go +++ b/plugins/outputs/microsoft_fabric/event_house.go @@ -5,9 +5,10 @@ import ( "time" "github.com/Azure/azure-kusto-go/kusto/ingest" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/config" - adx "github.com/influxdata/telegraf/plugins/common/adx" + "github.com/influxdata/telegraf/plugins/common/adx" "github.com/influxdata/telegraf/plugins/serializers/json" ) diff --git a/plugins/outputs/microsoft_fabric/event_stream.go b/plugins/outputs/microsoft_fabric/event_stream.go index 179c6f3b371cb..fad32023fa3fb 100644 --- a/plugins/outputs/microsoft_fabric/event_stream.go +++ b/plugins/outputs/microsoft_fabric/event_stream.go @@ -3,7 +3,6 @@ package microsoft_fabric import ( "context" - _ "embed" "errors" "fmt" "time" @@ -29,10 +28,14 @@ type EventStream struct { } func (e *EventStream) Init() error { - e.serializer = &json.Serializer{ + serializer := &json.Serializer{ TimestampUnits: config.Duration(time.Nanosecond), TimestampFormat: time.RFC3339Nano, } + if err := serializer.Init(); err != nil { + return err + } + e.serializer = serializer if e.MaxMessageSize > 0 { e.options.MaxBytes = uint64(e.MaxMessageSize) } diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric.go b/plugins/outputs/microsoft_fabric/microsoft_fabric.go index d47e3427e151b..e656d87ac5dc3 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric.go @@ -9,7 +9,7 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/config" - adx "github.com/influxdata/telegraf/plugins/common/adx" + "github.com/influxdata/telegraf/plugins/common/adx" "github.com/influxdata/telegraf/plugins/outputs" ) @@ -41,7 +41,7 @@ func (m *MicrosoftFabric) Connect() error { } // SampleConfig implements telegraf.Output. -func (m *MicrosoftFabric) SampleConfig() string { +func (*MicrosoftFabric) SampleConfig() string { return sampleConfig } @@ -110,7 +110,6 @@ func isKustoEndpoint(endpoint string) bool { } func init() { - outputs.Add("microsoft_fabric", func() telegraf.Output { return &MicrosoftFabric{ Eventstream: &EventStream{ diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go index 66ba99e212509..b66cb0330317f 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go @@ -4,14 +4,14 @@ import ( "testing" "time" - "github.com/influxdata/telegraf/config" - "github.com/influxdata/telegraf/plugins/common/adx" - - "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/plugins/common/adx" + "github.com/influxdata/telegraf/testutil" ) type MockOutput struct { @@ -51,6 +51,12 @@ func TestMicrosoftFabric_Connect(t *testing.T) { mockOutput.AssertExpectations(t) } +func TestMicrosoftFabric_Connect_Err(t *testing.T) { + plugin := MicrosoftFabric{} + err := plugin.Connect() + require.Equal(t, "no active plugin to connect", err.Error()) +} + func TestMicrosoftFabric_Close(t *testing.T) { mockOutput := new(MockOutput) mockOutput.On("Close").Return(nil) @@ -64,6 +70,12 @@ func TestMicrosoftFabric_Close(t *testing.T) { mockOutput.AssertExpectations(t) } +func TestMicrosoftFabric_Close_Err(t *testing.T) { + plugin := MicrosoftFabric{} + err := plugin.Close() + require.Equal(t, "no active plugin to close", err.Error()) +} + func TestMicrosoftFabric_Write(t *testing.T) { mockOutput := new(MockOutput) mockOutput.On("Write", mock.Anything).Return(nil) @@ -81,6 +93,17 @@ func TestMicrosoftFabric_Write(t *testing.T) { mockOutput.AssertExpectations(t) } +func TestMicrosoftFabric_Write_Err(t *testing.T) { + plugin := MicrosoftFabric{} + + metrics := []telegraf.Metric{ + testutil.TestMetric(1.0, "test_metric"), + } + + err := plugin.Write(metrics) + require.Equal(t, "no active plugin to write to", err.Error()) +} + func TestIsKustoEndpoint(t *testing.T) { testCases := []struct { name string From ca7ece3253caa4bb7dd5d4011c7265f00d3fdfeb Mon Sep 17 00:00:00 2001 From: asaharn Date: Mon, 21 Apr 2025 09:11:13 +0530 Subject: [PATCH 04/28] comitted readme --- plugins/outputs/microsoft_fabric/readme.md | 166 +++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 plugins/outputs/microsoft_fabric/readme.md diff --git a/plugins/outputs/microsoft_fabric/readme.md b/plugins/outputs/microsoft_fabric/readme.md new file mode 100644 index 0000000000000..0af3659b992df --- /dev/null +++ b/plugins/outputs/microsoft_fabric/readme.md @@ -0,0 +1,166 @@ +# Microsoft Fabric Output Plugin + +This plugin writes metrics to [Real time analytics in Fabric][fabric] services. + +[fabric]: https://learn.microsoft.com/en-us/fabric/real-time-analytics/overview + +## Global configuration options + +In addition to the plugin-specific configuration settings, plugins support +additional global and plugin configuration settings. These settings are used to +modify metrics, tags, and field or create aliases and configure ordering, etc. +See the [CONFIGURATION.md][CONFIGURATION.md] for more details. + +[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins + +## Configuration + +```toml +# Sends metrics to Microsoft Fabric +[[outputs.microsoft_fabric]] + ## The URI property of the resource on Azure + ## ex: connection_string = "data source=https://myadxresource.australiasoutheast.kusto.windows.net" + ## ex: connection_string = "Endpoint=sb://namespace.servicebus.windows.net/;*****;EntityPath=hubName" + connection_string = "" + + + [outputs.microsoft_fabric.eh_conf] + ## The Eventhouse database that the metrics will be ingested into. + ## The plugin will NOT generate this database automatically, it's expected that this database already exists before ingestion. + ## ex: "exampledatabase" + database = "" + + ## Timeout for Eventhouse operations (defaults to 30s) + # timeout = "20s" + + ## Type of metrics grouping used when pushing to Eventhouse. + ## Default is "TablePerMetric" for one table per different metric. + ## For more information, please check the plugin README. + # metrics_grouping_type = "TablePerMetric" + + ## Name of the single table to store all the metrics (Only needed if metrics_grouping_type is "SingleTable"). + # table_name = "" + + ## Creates tables and relevant mapping if set to true(default). + ## Skips table and mapping creation if set to false, this is useful for running Telegraf with the lowest possible permissions i.e. table ingestor role. + # create_tables = true + + ## Ingestion method to use. + ## Available options are + ## - managed -- streaming ingestion with fallback to batched ingestion or the "queued" method below + ## - queued -- queue up metrics data and process sequentially + # ingestion_type = "queued" + + [outputs.microsoft_fabric.es_conf] + ## The full connection string to the Event stream (required) + ## The shared access key must have "Send" permissions on the target Event stream. + + ## Client timeout (defaults to 30s) + # timeout = "30s" + + ## Partition key + ## Metric tag or field name to use for the event partition key. The value of + ## this tag or field is set as the key for events if it exists. If both, tag + ## and field, exist the tag is preferred. + # partition_key = "" + + ## Set the maximum batch message size in bytes + ## The allowable size depends on the Event stream tier + ## See: https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas#basic-vs-standard-vs-premium-vs-dedicated-tiers + ## Setting this to 0 means using the default size from the Azure Event streams Client library (1000000 bytes) + # max_message_size = 1000000 + + ## Data format to output. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md + data_format = "json" +``` + +## Description + +The `microsoft_fabric` output plugin sends metrics to Microsoft Fabric, +a scalable data platform for real-time analytics. +This plugin allows you to leverage Microsoft Fabric's +capabilities to store and analyze your Telegraf metrics. +Following are the currently supported datastores: + +### Eventhouse + +Eventhouse is a high-performance, scalable data store designed for + real-time analytics. It allows you to ingest, store, and query large + volumes of data with low latency. For more information, visit the + [Eventhouse documentation]( + https://learn.microsoft.com/fabric/real-time-intelligence/eventhouse + ). + +```toml +[outputs.microsoft_fabric.eventhouse_conf] + [outputs.microsoft_fabric.eventhouse_conf.cluster_config] + ## The Eventhouse database that the metrics will be ingested into. + ## The plugin will NOT generate this database automatically, it's expected that this database already exists before ingestion. + ## ex: "exampledatabase" + database = "" + + ## Timeout for Eventhouse operations + # timeout = "20s" + + ## Type of metrics grouping used when pushing to Eventhouse. + ## Default is "TablePerMetric" for one table per different metric. + ## For more information, please check the plugin README. + # metrics_grouping_type = "TablePerMetric" + + ## Name of the single table to store all the metrics (Only needed if metrics_grouping_type is "SingleTable"). + # table_name = "" + + ## Creates tables and relevant mapping if set to true(default). + ## Skips table and mapping creation if set to false, this is useful for running Telegraf with the lowest possible permissions i.e. table ingestor role. + # create_tables = true + + ## Ingestion method to use. + ## Available options are + ## - managed -- streaming ingestion with fallback to batched ingestion or the "queued" method below + ## - queued -- queue up metrics data and process sequentially + # ingestion_type = "queued" + +``` + +More about the eventhouse configuration properties +can be found [here](../azure_data_explorer/README.md#metrics-grouping) + +### Eventstream + +The eventstreams feature in the Microsoft Fabric Real-Time Intelligence +experience lets you bring real-time events into Fabric, transform them, +and then route them to various destinations without writing any code (no-code). +For more information, visit the [Eventstream documentation][]. + +[Eventstream documentation]: https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/overview?tabs=enhancedcapabilities + +```toml +[outputs.microsoft_fabric.eventstream_conf] + ## The full connection string to the Event stream (required) + ## The shared access key must have "Send" permissions on the target Event stream. + + ## Client timeout (defaults to 30s) + # timeout = "30s" + + ## Partition key + ## Metric tag or field name to use for the event partition key. The value of + ## this tag or field is set as the key for events if it exists. If both, tag + ## and field, exist the tag is preferred. + # partition_key = "" + + ## Set the maximum batch message size in bytes + ## The allowable size depends on the Event stream tier + ## See: https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas#basic-vs-standard-vs-premium-vs-dedicated-tiers + ## Setting this to 0 means using the default size from the Azure Event streams Client library (1000000 bytes) + # max_message_size = 1000000 + + ## Data format to output. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md + data_format = "json" + +``` From 38b90618e9d873c5623f0161674deb74c45bb992 Mon Sep 17 00:00:00 2001 From: asaharn Date: Mon, 21 Apr 2025 09:26:05 +0530 Subject: [PATCH 05/28] rename --- plugins/outputs/microsoft_fabric/{readme.md => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename plugins/outputs/microsoft_fabric/{readme.md => README.md} (100%) diff --git a/plugins/outputs/microsoft_fabric/readme.md b/plugins/outputs/microsoft_fabric/README.md similarity index 100% rename from plugins/outputs/microsoft_fabric/readme.md rename to plugins/outputs/microsoft_fabric/README.md From 360ada32e0017a02918f400bc84c741d7de84666 Mon Sep 17 00:00:00 2001 From: asaharn Date: Tue, 6 May 2025 19:44:38 +0530 Subject: [PATCH 06/28] Updated as per code review comments --- .../microsoft_fabric/EVENTHOUSE_CONFIGS.md | 261 ++++++++++++++++++ plugins/outputs/microsoft_fabric/README.md | 198 +++++-------- .../outputs/microsoft_fabric/event_house.go | 6 +- .../outputs/microsoft_fabric/event_stream.go | 15 +- .../microsoft_fabric/microsoft_fabric.go | 72 +++-- plugins/outputs/microsoft_fabric/sample.conf | 111 ++++---- 6 files changed, 429 insertions(+), 234 deletions(-) create mode 100644 plugins/outputs/microsoft_fabric/EVENTHOUSE_CONFIGS.md diff --git a/plugins/outputs/microsoft_fabric/EVENTHOUSE_CONFIGS.md b/plugins/outputs/microsoft_fabric/EVENTHOUSE_CONFIGS.md new file mode 100644 index 0000000000000..a96b6153c6b63 --- /dev/null +++ b/plugins/outputs/microsoft_fabric/EVENTHOUSE_CONFIGS.md @@ -0,0 +1,261 @@ +# Eventhouse Configuration + +```toml @sample.conf +[outputs.microsoft_fabric.eventstream] + ## The full connection string to the Event Hub (required) + ## The shared access key must have "Send" permissions on the target Event Hub. + + ## Client timeout + # timeout = "30s" + + ## Partition key + ## Metric tag or field name to use for the event partition key. The value of + ## this tag or field is set as the key for events if it exists. If both, tag + ## and field, exist the tag is preferred. + # partition_key = "" + + ## Set the maximum batch message size in bytes + ## The allowable size depends on the Event Hub tier; not setting this option or setting + ## it to zero will use the default size of the Azure Event Hubs Client library. See + ## https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas#basic-vs-standard-vs-premium-vs-dedicated-tiers + ## for the allowable size of your tier. + # max_message_size = "0B" + + ## Data format to output. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md + data_format = "json" + +``` + +## Metrics Grouping + +Metrics can be grouped in two ways to be sent to Azure Data Explorer. To specify +which metric grouping type the plugin should use, the respective value should be +given to the `metrics_grouping_type` in the config file. If no value is given to +`metrics_grouping_type`, by default, the metrics will be grouped using +`TablePerMetric`. + +### TablePerMetric + +The plugin will group the metrics by the metric name, and will send each group +of metrics to an Azure Data Explorer table. If the table doesn't exist the +plugin will create the table, if the table exists then the plugin will try to +merge the Telegraf metric schema to the existing table. For more information +about the merge process check the [`.create-merge` documentation][create-merge]. + +The table name will match the `name` property of the metric, this means that the +name of the metric should comply with the Azure Data Explorer table naming +constraints in case you plan to add a prefix to the metric name. + +[create-merge]: https://docs.microsoft.com/en-us/azure/data-explorer/kusto/management/create-merge-table-command + +### SingleTable + +The plugin will send all the metrics received to a single Azure Data Explorer +table. The name of the table must be supplied via `table_name` in the config +file. If the table doesn't exist the plugin will create the table, if the table +exists then the plugin will try to merge the Telegraf metric schema to the +existing table. For more information about the merge process check the +[`.create-merge` documentation][create-merge]. + +## Tables Schema + +The schema of the Azure Data Explorer table will match the structure of the +Telegraf `Metric` object. The corresponding Azure Data Explorer command +generated by the plugin would be like the following: + +```text +.create-merge table ['table-name'] (['fields']:dynamic, ['name']:string, ['tags']:dynamic, ['timestamp']:datetime) +``` + +The corresponding table mapping would be like the following: + +```text +.create-or-alter table ['table-name'] ingestion json mapping 'table-name_mapping' '[{"column":"fields", "Properties":{"Path":"$[\'fields\']"}},{"column":"name", "Properties":{"Path":"$[\'name\']"}},{"column":"tags", "Properties":{"Path":"$[\'tags\']"}},{"column":"timestamp", "Properties":{"Path":"$[\'timestamp\']"}}]' +``` + +**Note**: This plugin will automatically create Azure Data Explorer tables and +corresponding table mapping as per the above mentioned commands. + +## Ingestion type + +**Note**: +[Streaming ingestion](https://aka.ms/AAhlg6s) +has to be enabled on ADX [configure the ADX cluster] +in case of `managed` option. +Refer the query below to check if streaming is enabled + +```kql +.show database policy streamingingestion +``` + +## Authentication + +### Supported Authentication Methods + +This plugin provides several types of authentication. The plugin will check the +existence of several specific environment variables, and consequently will +choose the right method. + +These methods are: + +1. AAD Application Tokens (Service Principals with secrets or certificates). + + For guidance on how to create and register an App in Azure Active Directory + check [this article][register], and for more information on the Service + Principals check [this article][principal]. + +2. AAD User Tokens + + - Allows Telegraf to authenticate like a user. This method is mainly used + for development purposes only. + +3. Managed Service Identity (MSI) token + + - If you are running Telegraf from Azure VM or infrastructure, then this is + the preferred authentication method. + +[register]: https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app#register-an-application + +[principal]: https://docs.microsoft.com/en-us/azure/active-directory/develop/app-objects-and-service-principals + +Whichever method, the designated Principal needs to be assigned the `Database +User` role on the Database level in the Azure Data Explorer. This role will +allow the plugin to create the required tables and ingest data into it. If +`create_tables=false` then the designated principal only needs the `Database +Ingestor` role at least. + +### Configurations of the chosen Authentication Method + +The plugin will authenticate using the first available of the following +configurations, **it's important to understand that the assessment, and +consequently choosing the authentication method, will happen in order as +below**: + +1. **Client Credentials**: Azure AD Application ID and Secret. + + Set the following environment variables: + + - `AZURE_TENANT_ID`: Specifies the Tenant to which to authenticate. + - `AZURE_CLIENT_ID`: Specifies the app client ID to use. + - `AZURE_CLIENT_SECRET`: Specifies the app secret to use. + +2. **Client Certificate**: Azure AD Application ID and X.509 Certificate. + + - `AZURE_TENANT_ID`: Specifies the Tenant to which to authenticate. + - `AZURE_CLIENT_ID`: Specifies the app client ID to use. + - `AZURE_CERTIFICATE_PATH`: Specifies the certificate Path to use. + - `AZURE_CERTIFICATE_PASSWORD`: Specifies the certificate password to use. + +3. **Resource Owner Password**: Azure AD User and Password. This grant type is + *not recommended*, use device login instead if you need interactive login. + + - `AZURE_TENANT_ID`: Specifies the Tenant to which to authenticate. + - `AZURE_CLIENT_ID`: Specifies the app client ID to use. + - `AZURE_USERNAME`: Specifies the username to use. + - `AZURE_PASSWORD`: Specifies the password to use. + +4. **Azure Managed Service Identity**: Delegate credential management to the + platform. Requires that code is running in Azure, e.g. on a VM. All + configuration is handled by Azure. See [Azure Managed Service Identity][msi] + for more details. Only available when using the [Azure Resource + Manager][arm]. + +[msi]: https://docs.microsoft.com/en-us/azure/active-directory/msi-overview +[arm]: https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-overview + +## Querying data collected in Azure Data Explorer + +Examples of data transformations and queries that would be useful to gain +insights - + +### Using SQL input plugin + +Sample SQL metrics data - + +name | tags | timestamp | fields +-----|------|-----------|------- +sqlserver_database_io|{"database_name":"azure-sql-db2","file_type":"DATA","host":"adx-vm","logical_filename":"tempdev","measurement_db_type":"AzureSQLDB","physical_filename":"tempdb.mdf","replica_updateability":"READ_WRITE","sql_instance":"adx-sql-server"}|2021-09-09T13:51:20Z|{"current_size_mb":16,"database_id":2,"file_id":1,"read_bytes":2965504,"read_latency_ms":68,"reads":47,"rg_read_stall_ms":42,"rg_write_stall_ms":0,"space_used_mb":0,"write_bytes":1220608,"write_latency_ms":103,"writes":149} +sqlserver_waitstats|{"database_name":"azure-sql-db2","host":"adx-vm","measurement_db_type":"AzureSQLDB","replica_updateability":"READ_WRITE","sql_instance":"adx-sql-server","wait_category":"Worker Thread","wait_type":"THREADPOOL"}|2021-09-09T13:51:20Z|{"max_wait_time_ms":15,"resource_wait_ms":4469,"signal_wait_time_ms":0,"wait_time_ms":4469,"waiting_tasks_count":1464} + +Since collected metrics object is of complex type so "fields" and "tags" are +stored as dynamic data type, multiple ways to query this data- + +1. Query JSON attributes directly: Azure Data Explorer provides an ability to + query JSON data in raw format without parsing it, so JSON attributes can be + queried directly in following way: + + ```text + Tablename + | where name == "sqlserver_azure_db_resource_stats" and todouble(fields.avg_cpu_percent) > 7 + ``` + + ```text + Tablename + | distinct tostring(tags.database_name) + ``` + + **Note** - This approach could have performance impact in case of large + volumes of data, use below mentioned approach for such cases. + +1. Use [Update + policy](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/management/updatepolicy)**: + Transform dynamic data type columns using update policy. This is the + recommended performant way for querying over large volumes of data compared + to querying directly over JSON attributes: + + ```json + // Function to transform data + .create-or-alter function Transform_TargetTableName() { + SourceTableName + | mv-apply fields on (extend key = tostring(bag_keys(fields)[0])) + | project fieldname=key, value=todouble(fields[key]), name, tags, timestamp + } + + // Create destination table with above query's results schema (if it doesn't exist already) + .set-or-append TargetTableName <| Transform_TargetTableName() | limit 0 + + // Apply update policy on destination table + .alter table TargetTableName policy update + @'[{"IsEnabled": true, "Source": "SourceTableName", "Query": "Transform_TargetTableName()", "IsTransactional": true, "PropagateIngestionProperties": false}]' + ``` + +### Using syslog input plugin + +Sample syslog data - + +name | tags | timestamp | fields +-----|------|-----------|------- +syslog|{"appname":"azsecmond","facility":"user","host":"adx-linux-vm","hostname":"adx-linux-vm","severity":"info"}|2021-09-20T14:36:44Z|{"facility_code":1,"message":" 2021/09/20 14:36:44.890110 Failed to connect to mdsd: dial unix /var/run/mdsd/default_djson.socket: connect: no such file or directory","procid":"2184","severity_code":6,"timestamp":"1632148604890477000","version":1} +syslog|{"appname":"CRON","facility":"authpriv","host":"adx-linux-vm","hostname":"adx-linux-vm","severity":"info"}|2021-09-20T14:37:01Z|{"facility_code":10,"message":" pam_unix(cron:session): session opened for user root by (uid=0)","procid":"26446","severity_code":6,"timestamp":"1632148621120781000","version":1} + +There are multiple ways to flatten dynamic columns using 'extend' or +'bag_unpack' operator. You can use either of these ways in above mentioned +update policy function - 'Transform_TargetTableName()' + +- Use + [extend](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/extendoperator) + operator - This is the recommended approach compared to 'bag_unpack' as it is + faster and robust. Even if schema changes, it will not break queries or + dashboards. + + ```text + Tablenmae + | extend facility_code=toint(fields.facility_code), message=tostring(fields.message), procid= tolong(fields.procid), severity_code=toint(fields.severity_code), + SysLogTimestamp=unixtime_nanoseconds_todatetime(tolong(fields.timestamp)), version= todouble(fields.version), + appname= tostring(tags.appname), facility= tostring(tags.facility),host= tostring(tags.host), hostname=tostring(tags.hostname), severity=tostring(tags.severity) + | project-away fields, tags + ``` + +- Use [bag_unpack + plugin](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/bag-unpackplugin) + to unpack the dynamic type columns automatically. This method could lead to + issues if source schema changes as its dynamically expanding columns. + + ```text + Tablename + | evaluate bag_unpack(tags, columnsConflict='replace_source') + | evaluate bag_unpack(fields, columnsConflict='replace_source') + ``` diff --git a/plugins/outputs/microsoft_fabric/README.md b/plugins/outputs/microsoft_fabric/README.md index 0af3659b992df..19a4c7711400d 100644 --- a/plugins/outputs/microsoft_fabric/README.md +++ b/plugins/outputs/microsoft_fabric/README.md @@ -2,6 +2,10 @@ This plugin writes metrics to [Real time analytics in Fabric][fabric] services. +⭐ Telegraf v1.35.0 +🏷️ datastore +💻 all + [fabric]: https://learn.microsoft.com/en-us/fabric/real-time-analytics/overview ## Global configuration options @@ -18,149 +22,83 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. ```toml # Sends metrics to Microsoft Fabric [[outputs.microsoft_fabric]] - ## The URI property of the resource on Azure - ## ex: connection_string = "data source=https://myadxresource.australiasoutheast.kusto.windows.net" - ## ex: connection_string = "Endpoint=sb://namespace.servicebus.windows.net/;*****;EntityPath=hubName" + ## The URI property of the Eventhouse resource on Azure + ## ex: connection_string = "Data Source=https://myadxresource.australiasoutheast.kusto.windows.net" connection_string = "" - [outputs.microsoft_fabric.eh_conf] - ## The Eventhouse database that the metrics will be ingested into. - ## The plugin will NOT generate this database automatically, it's expected that this database already exists before ingestion. - ## ex: "exampledatabase" - database = "" - - ## Timeout for Eventhouse operations (defaults to 30s) - # timeout = "20s" - - ## Type of metrics grouping used when pushing to Eventhouse. - ## Default is "TablePerMetric" for one table per different metric. - ## For more information, please check the plugin README. - # metrics_grouping_type = "TablePerMetric" - - ## Name of the single table to store all the metrics (Only needed if metrics_grouping_type is "SingleTable"). - # table_name = "" - - ## Creates tables and relevant mapping if set to true(default). - ## Skips table and mapping creation if set to false, this is useful for running Telegraf with the lowest possible permissions i.e. table ingestor role. - # create_tables = true - - ## Ingestion method to use. - ## Available options are - ## - managed -- streaming ingestion with fallback to batched ingestion or the "queued" method below - ## - queued -- queue up metrics data and process sequentially - # ingestion_type = "queued" - - [outputs.microsoft_fabric.es_conf] - ## The full connection string to the Event stream (required) - ## The shared access key must have "Send" permissions on the target Event stream. - - ## Client timeout (defaults to 30s) - # timeout = "30s" - - ## Partition key - ## Metric tag or field name to use for the event partition key. The value of - ## this tag or field is set as the key for events if it exists. If both, tag - ## and field, exist the tag is preferred. - # partition_key = "" - - ## Set the maximum batch message size in bytes - ## The allowable size depends on the Event stream tier - ## See: https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas#basic-vs-standard-vs-premium-vs-dedicated-tiers - ## Setting this to 0 means using the default size from the Azure Event streams Client library (1000000 bytes) - # max_message_size = 1000000 - - ## Data format to output. - ## Each data format has its own unique set of configuration options, read - ## more about them here: - ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md - data_format = "json" + ## Using this section the plugin will send metrics to an Eventhouse endpoint + ## for ingesting, storing, and querying large volumes of data with low latency. + [outputs.microsoft_fabric.eventhouse] + [outputs.microsoft_fabric.eventhouse.cluster_config] + ## Database metrics will be written to + ## NOTE: The plugin will NOT generate the database. It is expected the database already exists. + database = "" + + ## Timeout for Eventhouse operations + # timeout = "20s" + + ## Type of metrics grouping; available options are: + ## tablepermetric -- for one table per distinct metric + ## singletable -- for writing all metrics to the same table + # metrics_grouping_type = "tablepermetric" + + # Name of the table to store metrics + ## NOTE: This option is only used for "singletable" metrics grouping + # table_name = "" + + ## Creates tables and relevant mapping + ## Disable when running with the lowest possible permissions i.e. table ingestor role. + # create_tables = true + + ## Ingestion method to use; available options are + ## - managed -- streaming ingestion with fallback to batched ingestion or the "queued" method below + ## - queued -- queue up metrics data and process sequentially + # ingestion_type = "queued" + + ## Using this section the plugin will send metrics to an EventStream endpoint + ## for transforming and routing metrics to various destinations without writing + ## any code. + [outputs.microsoft_fabric.eventstream] + ## The full connection string to the Event Hub (required) + ## The shared access key must have "Send" permissions on the target Event Hub. + + ## Client timeout + # timeout = "30s" + + ## Partition key + ## Metric tag or field name to use for the event partition key. The value of + ## this tag or field is set as the key for events if it exists. If both, tag + ## and field, exist the tag is preferred. + # partition_key = "" + + ## Set the maximum batch message size in bytes + ## The allowable size depends on the Event Hub tier; not setting this option or setting + ## it to zero will use the default size of the Azure Event Hubs Client library. See + ## https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas#basic-vs-standard-vs-premium-vs-dedicated-tiers + ## for the allowable size of your tier. + # max_message_size = "0B" + + ## Data format to output. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md + data_format = "json" ``` -## Description - -The `microsoft_fabric` output plugin sends metrics to Microsoft Fabric, -a scalable data platform for real-time analytics. -This plugin allows you to leverage Microsoft Fabric's -capabilities to store and analyze your Telegraf metrics. -Following are the currently supported datastores: - -### Eventhouse +### EventHouse -Eventhouse is a high-performance, scalable data store designed for - real-time analytics. It allows you to ingest, store, and query large - volumes of data with low latency. For more information, visit the - [Eventhouse documentation]( - https://learn.microsoft.com/fabric/real-time-intelligence/eventhouse - ). - -```toml -[outputs.microsoft_fabric.eventhouse_conf] - [outputs.microsoft_fabric.eventhouse_conf.cluster_config] - ## The Eventhouse database that the metrics will be ingested into. - ## The plugin will NOT generate this database automatically, it's expected that this database already exists before ingestion. - ## ex: "exampledatabase" - database = "" - - ## Timeout for Eventhouse operations - # timeout = "20s" - - ## Type of metrics grouping used when pushing to Eventhouse. - ## Default is "TablePerMetric" for one table per different metric. - ## For more information, please check the plugin README. - # metrics_grouping_type = "TablePerMetric" - - ## Name of the single table to store all the metrics (Only needed if metrics_grouping_type is "SingleTable"). - # table_name = "" - - ## Creates tables and relevant mapping if set to true(default). - ## Skips table and mapping creation if set to false, this is useful for running Telegraf with the lowest possible permissions i.e. table ingestor role. - # create_tables = true - - ## Ingestion method to use. - ## Available options are - ## - managed -- streaming ingestion with fallback to batched ingestion or the "queued" method below - ## - queued -- queue up metrics data and process sequentially - # ingestion_type = "queued" +The microsoft_fabric output plugin sends metrics to Microsoft Fabric, a scalable data platform for real-time analytics. This plugin allows you to leverage Microsoft Fabric's capabilities to store and analyze your Telegraf metrics. Eventhouse is a high-performance, scalable data store designed for real-time analytics. It allows you to ingest, store, and query large volumes of data with low latency. For more information, visit the Eventhouse documentation. -``` More about the eventhouse configuration properties -can be found [here](../azure_data_explorer/README.md#metrics-grouping) +can be found [here](./EVENTHOUSE_CONFIGS.md) ### Eventstream The eventstreams feature in the Microsoft Fabric Real-Time Intelligence experience lets you bring real-time events into Fabric, transform them, and then route them to various destinations without writing any code (no-code). -For more information, visit the [Eventstream documentation][]. - -[Eventstream documentation]: https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/overview?tabs=enhancedcapabilities +For more information, visit the [Eventstream documentation][eventstream_docs]. -```toml -[outputs.microsoft_fabric.eventstream_conf] - ## The full connection string to the Event stream (required) - ## The shared access key must have "Send" permissions on the target Event stream. - - ## Client timeout (defaults to 30s) - # timeout = "30s" - - ## Partition key - ## Metric tag or field name to use for the event partition key. The value of - ## this tag or field is set as the key for events if it exists. If both, tag - ## and field, exist the tag is preferred. - # partition_key = "" - - ## Set the maximum batch message size in bytes - ## The allowable size depends on the Event stream tier - ## See: https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas#basic-vs-standard-vs-premium-vs-dedicated-tiers - ## Setting this to 0 means using the default size from the Azure Event streams Client library (1000000 bytes) - # max_message_size = 1000000 - - ## Data format to output. - ## Each data format has its own unique set of configuration options, read - ## more about them here: - ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md - data_format = "json" - -``` +[eventstream_docs]: https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/overview?tabs=enhancedcapabilities diff --git a/plugins/outputs/microsoft_fabric/event_house.go b/plugins/outputs/microsoft_fabric/event_house.go index d29431083da94..657add09ea54b 100644 --- a/plugins/outputs/microsoft_fabric/event_house.go +++ b/plugins/outputs/microsoft_fabric/event_house.go @@ -32,10 +32,12 @@ func (e *EventHouse) Init() error { } func (e *EventHouse) Connect() error { - var err error - if e.client, err = e.Config.NewClient("Kusto.Telegraf", e.log); err != nil { + client, err := e.Config.NewClient("Kusto.Telegraf", e.log) + if err != nil { return fmt.Errorf("creating new client failed: %w", err) } + e.client = client + return nil } diff --git a/plugins/outputs/microsoft_fabric/event_stream.go b/plugins/outputs/microsoft_fabric/event_stream.go index fad32023fa3fb..9aac264894f59 100644 --- a/plugins/outputs/microsoft_fabric/event_stream.go +++ b/plugins/outputs/microsoft_fabric/event_stream.go @@ -70,10 +70,13 @@ func (e *EventStream) SetSerializer(serializer telegraf.Serializer) { } func (e *EventStream) Write(metrics []telegraf.Metric) error { + // This context is only used for creating the batches which should not timeout as this is + // not an I/O operation. Therefore avoid setting a timeout here. ctx := context.Background() batchOptions := e.options batches := make(map[string]*azeventhubs.EventDataBatch) + // Cant use `for _, m := range metrics` as we need to move back when a new batch needs to be created for i := 0; i < len(metrics); i++ { m := metrics[i] @@ -87,18 +90,16 @@ func (e *EventStream) Write(metrics []telegraf.Metric) error { // Get the batcher for the chosen partition partition := "" - batchOptions.PartitionKey = nil if e.PartitionKey != "" { if key, ok := m.GetTag(e.PartitionKey); ok { partition = key - batchOptions.PartitionKey = &partition } else if key, ok := m.GetField(e.PartitionKey); ok { if k, ok := key.(string); ok { partition = k - batchOptions.PartitionKey = &partition } } } + batchOptions.PartitionKey = &partition if _, found := batches[partition]; !found { batches[partition], err = e.client.NewEventDataBatch(ctx, &batchOptions) if err != nil { @@ -124,7 +125,7 @@ func (e *EventStream) Write(metrics []telegraf.Metric) error { e.log.Tracef("metric: %+v", m) continue } - if err := e.send(batches[partition]); err != nil { + if err := e.send(ctx, batches[partition]); err != nil { return fmt.Errorf("sending batch for partition %q failed: %w", partition, err) } @@ -142,15 +143,15 @@ func (e *EventStream) Write(metrics []telegraf.Metric) error { if batch.NumBytes() == 0 { continue } - if err := e.send(batch); err != nil { + if err := e.send(ctx, batch); err != nil { return fmt.Errorf("sending batch for partition %q failed: %w", partition, err) } } return nil } -func (e *EventStream) send(batch *azeventhubs.EventDataBatch) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(e.Timeout)) +func (e *EventStream) send(ctx context.Context, batch *azeventhubs.EventDataBatch) error { + ctx, cancel := context.WithTimeout(ctx, time.Duration(e.Timeout)) defer cancel() return e.client.SendEventDataBatch(ctx, batch, nil) diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric.go b/plugins/outputs/microsoft_fabric/microsoft_fabric.go index e656d87ac5dc3..0ac7d0f8fb2d8 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric.go @@ -4,6 +4,7 @@ package microsoft_fabric import ( _ "embed" "errors" + "fmt" "strings" "time" @@ -18,49 +19,22 @@ var sampleConfig string type MicrosoftFabric struct { ConnectionString string `toml:"connection_string"` + Eventhouse *EventHouse `toml:"eventhouse"` + Eventstream *EventStream `toml:"eventstream"` Log telegraf.Logger `toml:"-"` - Eventhouse *EventHouse `toml:"eventhouse_conf"` - Eventstream *EventStream `toml:"eventstream_conf"` - activePlugin FabricOutput -} -// Close implements telegraf.Output. -func (m *MicrosoftFabric) Close() error { - if m.activePlugin == nil { - return errors.New("no active plugin to close") - } - return m.activePlugin.Close() + activePlugin FabricOutput } -// Connect implements telegraf.Output. -func (m *MicrosoftFabric) Connect() error { - if m.activePlugin == nil { - return errors.New("no active plugin to connect") - } - return m.activePlugin.Connect() -} - -// SampleConfig implements telegraf.Output. func (*MicrosoftFabric) SampleConfig() string { return sampleConfig } -// Write implements telegraf.Output. -func (m *MicrosoftFabric) Write(metrics []telegraf.Metric) error { - if m.activePlugin == nil { - return errors.New("no active plugin to write to") - } - return m.activePlugin.Write(metrics) -} - func (m *MicrosoftFabric) Init() error { connectionString := m.ConnectionString if connectionString == "" { - return errors.New("endpoint must not be empty. For EventHouse refer : " + - "https://learn.microsoft.com/kusto/api/connection-strings/kusto?view=microsoft-fabric " + - "for EventStream refer : https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/add-manage-eventstream-sources" + - "?pivots=enhanced-capabilities") + return errors.New("endpoint must not be empty") } if strings.HasPrefix(connectionString, "Endpoint=sb") { @@ -68,9 +42,8 @@ func (m *MicrosoftFabric) Init() error { m.Eventstream.connectionString = connectionString m.Eventstream.log = m.Log - err := m.Eventstream.Init() - if err != nil { - return errors.New("error initializing EventStream plugin: " + err.Error()) + if err := m.Eventstream.Init(); err != nil { + return fmt.Errorf("initializing EventStream output failed: %w", err) } m.activePlugin = m.Eventstream } else if isKustoEndpoint(strings.ToLower(connectionString)) { @@ -78,20 +51,37 @@ func (m *MicrosoftFabric) Init() error { // Setting up the AzureDataExplorer plugin initial properties m.Eventhouse.Config.Endpoint = connectionString m.Eventhouse.log = m.Log - err := m.Eventhouse.Init() - if err != nil { - return errors.New("error initializing EventHouse plugin: " + err.Error()) + if err := m.Eventhouse.Init(); err != nil { + return fmt.Errorf("initializing EventHouse output failed: %w", err) } m.activePlugin = m.Eventhouse } else { - return errors.New("invalid connection string. For EventHouse refer : " + - "https://learn.microsoft.com/kusto/api/connection-strings/kusto?view=microsoft-fabric" + - " for EventStream refer : https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/" + - "add-manage-eventstream-sources?pivots=enhanced-capabilities") + return errors.New("invalid connection string") } return nil } +func (m *MicrosoftFabric) Close() error { + if m.activePlugin == nil { + return errors.New("no active plugin to close") + } + return m.activePlugin.Close() +} + +func (m *MicrosoftFabric) Connect() error { + if m.activePlugin == nil { + return errors.New("no active plugin to connect") + } + return m.activePlugin.Connect() +} + +func (m *MicrosoftFabric) Write(metrics []telegraf.Metric) error { + if m.activePlugin == nil { + return errors.New("no active plugin to write to") + } + return m.activePlugin.Write(metrics) +} + func isKustoEndpoint(endpoint string) bool { prefixes := []string{ "data source=", diff --git a/plugins/outputs/microsoft_fabric/sample.conf b/plugins/outputs/microsoft_fabric/sample.conf index f49b45a0d7cbf..23a8ac906b746 100644 --- a/plugins/outputs/microsoft_fabric/sample.conf +++ b/plugins/outputs/microsoft_fabric/sample.conf @@ -5,57 +5,60 @@ connection_string = "" - [outputs.microsoft_fabric.eventhouse_conf] - [outputs.microsoft_fabric.eventhouse_conf.cluster_config] - ## The Eventhouse database that the metrics will be ingested into. - ## The plugin will NOT generate this database automatically, it's expected that this database already exists before ingestion. - ## ex: "exampledatabase" - database = "" - - ## Timeout for Eventhouse operations - # timeout = "20s" - - ## Type of metrics grouping used when pushing to Eventhouse. - ## Default is "TablePerMetric" for one table per different metric. - ## For more information, please check the plugin README. - # metrics_grouping_type = "TablePerMetric" - - ## Name of the single table to store all the metrics (Only needed if metrics_grouping_type is "SingleTable"). - # table_name = "" - - ## Creates tables and relevant mapping if set to true(default). - ## Skips table and mapping creation if set to false, this is useful for running Telegraf with the lowest possible permissions i.e. table ingestor role. - # create_tables = true - - ## Ingestion method to use. - ## Available options are - ## - managed -- streaming ingestion with fallback to batched ingestion or the "queued" method below - ## - queued -- queue up metrics data and process sequentially - # ingestion_type = "queued" - - [outputs.microsoft_fabric.eventstream_conf] - ## The full connection string to the Event Hub (required) - ## The shared access key must have "Send" permissions on the target Event Hub. - - ## Client timeout (defaults to 30s) - # timeout = "30s" - - ## Partition key - ## Metric tag or field name to use for the event partition key. The value of - ## this tag or field is set as the key for events if it exists. If both, tag - ## and field, exist the tag is preferred. - # partition_key = "" - - ## Set the maximum batch message size in bytes - ## The allowable size depends on the Event Hub tier - ## See: https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas#basic-vs-standard-vs-premium-vs-dedicated-tiers - ## Setting this to 0 means using the default size from the Azure Event Hubs Client library (1000000 bytes) - # max_message_size = 1000000 - - ## Data format to output. - ## Each data format has its own unique set of configuration options, read - ## more about them here: - ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md - data_format = "json" - - \ No newline at end of file + ## Using this section the plugin will send metrics to an Eventhouse endpoint + ## for ingesting, storing, and querying large volumes of data with low latency. + [outputs.microsoft_fabric.eventhouse] + [outputs.microsoft_fabric.eventhouse.cluster_config] + ## Database metrics will be written to + ## NOTE: The plugin will NOT generate the database. It is expected the database already exists. + database = "" + + ## Timeout for Eventhouse operations + # timeout = "20s" + + ## Type of metrics grouping; available options are: + ## tablepermetric -- for one table per distinct metric + ## singletable -- for writing all metrics to the same table + # metrics_grouping_type = "tablepermetric" + + # Name of the table to store metrics + ## NOTE: This option is only used for "singletable" metrics grouping + # table_name = "" + + ## Creates tables and relevant mapping + ## Disable when running with the lowest possible permissions i.e. table ingestor role. + # create_tables = true + + ## Ingestion method to use; available options are + ## - managed -- streaming ingestion with fallback to batched ingestion or the "queued" method below + ## - queued -- queue up metrics data and process sequentially + # ingestion_type = "queued" + + ## Using this section the plugin will send metrics to an EventStream endpoint + ## for transforming and routing metrics to various destinations without writing + ## any code. + [outputs.microsoft_fabric.eventstream] + ## The full connection string to the Event Hub (required) + ## The shared access key must have "Send" permissions on the target Event Hub. + + ## Client timeout + # timeout = "30s" + + ## Partition key + ## Metric tag or field name to use for the event partition key. The value of + ## this tag or field is set as the key for events if it exists. If both, tag + ## and field, exist the tag is preferred. + # partition_key = "" + + ## Set the maximum batch message size in bytes + ## The allowable size depends on the Event Hub tier; not setting this option or setting + ## it to zero will use the default size of the Azure Event Hubs Client library. See + ## https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas#basic-vs-standard-vs-premium-vs-dedicated-tiers + ## for the allowable size of your tier. + # max_message_size = "0B" + + ## Data format to output. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md + data_format = "json" \ No newline at end of file From d7ed8e516589a12e744b0f682701ef1f4bb25802 Mon Sep 17 00:00:00 2001 From: asaharn Date: Tue, 6 May 2025 20:09:09 +0530 Subject: [PATCH 07/28] lint fix --- plugins/outputs/microsoft_fabric/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/outputs/microsoft_fabric/README.md b/plugins/outputs/microsoft_fabric/README.md index 19a4c7711400d..b6250f5fb96f4 100644 --- a/plugins/outputs/microsoft_fabric/README.md +++ b/plugins/outputs/microsoft_fabric/README.md @@ -90,7 +90,6 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. The microsoft_fabric output plugin sends metrics to Microsoft Fabric, a scalable data platform for real-time analytics. This plugin allows you to leverage Microsoft Fabric's capabilities to store and analyze your Telegraf metrics. Eventhouse is a high-performance, scalable data store designed for real-time analytics. It allows you to ingest, store, and query large volumes of data with low latency. For more information, visit the Eventhouse documentation. - More about the eventhouse configuration properties can be found [here](./EVENTHOUSE_CONFIGS.md) From c3286eaa085d3efd022f7bf36750bd1442e90339 Mon Sep 17 00:00:00 2001 From: asaharn Date: Tue, 6 May 2025 20:13:23 +0530 Subject: [PATCH 08/28] lint fix for readme --- plugins/outputs/microsoft_fabric/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/outputs/microsoft_fabric/README.md b/plugins/outputs/microsoft_fabric/README.md index b6250f5fb96f4..639ec34a108ef 100644 --- a/plugins/outputs/microsoft_fabric/README.md +++ b/plugins/outputs/microsoft_fabric/README.md @@ -88,7 +88,10 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. ### EventHouse -The microsoft_fabric output plugin sends metrics to Microsoft Fabric, a scalable data platform for real-time analytics. This plugin allows you to leverage Microsoft Fabric's capabilities to store and analyze your Telegraf metrics. Eventhouse is a high-performance, scalable data store designed for real-time analytics. It allows you to ingest, store, and query large volumes of data with low latency. For more information, visit the Eventhouse documentation. +The microsoft_fabric output plugin sends metrics to Microsoft Fabric, a scalable data platform for real-time analytics. +This plugin allows you to leverage Microsoft Fabric's capabilities to store and analyze your Telegraf metrics. +Eventhouse is a high-performance, scalable data store designed for real-time analytics. +It allows you to ingest, store, and query large volumes of data with low latency. For more information, visit the Eventhouse documentation. More about the eventhouse configuration properties can be found [here](./EVENTHOUSE_CONFIGS.md) From 8c60846e10efa7f327e32b6dae2e967338b056b8 Mon Sep 17 00:00:00 2001 From: asaharn Date: Tue, 6 May 2025 20:41:40 +0530 Subject: [PATCH 09/28] break lines for linting --- plugins/outputs/microsoft_fabric/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/README.md b/plugins/outputs/microsoft_fabric/README.md index 639ec34a108ef..7461f21ad2e33 100644 --- a/plugins/outputs/microsoft_fabric/README.md +++ b/plugins/outputs/microsoft_fabric/README.md @@ -88,9 +88,9 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. ### EventHouse -The microsoft_fabric output plugin sends metrics to Microsoft Fabric, a scalable data platform for real-time analytics. -This plugin allows you to leverage Microsoft Fabric's capabilities to store and analyze your Telegraf metrics. -Eventhouse is a high-performance, scalable data store designed for real-time analytics. +The microsoft_fabric output plugin sends metrics to Microsoft Fabric, a scalable data platform for real-time analytics.\ +This plugin allows you to leverage Microsoft Fabric's capabilities to store and analyze your Telegraf metrics.\ +Eventhouse is a high-performance, scalable data store designed for real-time analytics.\ It allows you to ingest, store, and query large volumes of data with low latency. For more information, visit the Eventhouse documentation. More about the eventhouse configuration properties @@ -103,4 +103,4 @@ experience lets you bring real-time events into Fabric, transform them, and then route them to various destinations without writing any code (no-code). For more information, visit the [Eventstream documentation][eventstream_docs]. -[eventstream_docs]: https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/overview?tabs=enhancedcapabilities +[eventstream_docs]: https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/overview?tabs=enhancedcapabilities From 334a46e62f018ab380ffa42865d44520449fff1a Mon Sep 17 00:00:00 2001 From: asaharn Date: Tue, 6 May 2025 20:46:25 +0530 Subject: [PATCH 10/28] reducing line sizes in md --- plugins/outputs/microsoft_fabric/README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/README.md b/plugins/outputs/microsoft_fabric/README.md index 7461f21ad2e33..705fbabc2bdc8 100644 --- a/plugins/outputs/microsoft_fabric/README.md +++ b/plugins/outputs/microsoft_fabric/README.md @@ -88,10 +88,13 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. ### EventHouse -The microsoft_fabric output plugin sends metrics to Microsoft Fabric, a scalable data platform for real-time analytics.\ -This plugin allows you to leverage Microsoft Fabric's capabilities to store and analyze your Telegraf metrics.\ -Eventhouse is a high-performance, scalable data store designed for real-time analytics.\ -It allows you to ingest, store, and query large volumes of data with low latency. For more information, visit the Eventhouse documentation. +The microsoft_fabric output plugin sends metrics to Microsoft Fabric, +a scalable data platform for real-time analytics. +This plugin allows you to leverage Microsoft Fabric's capabilities +to store and analyze your Telegraf metrics. Eventhouse is a high-performance, +scalable data store designed for real-time analytics. It allows you to ingest, +store, and query large volumes of data with low latency. For more information, +visit the Eventhouse documentation. More about the eventhouse configuration properties can be found [here](./EVENTHOUSE_CONFIGS.md) From f28d50bc5b38b527dc18a7d4412fdbcf7efcd22f Mon Sep 17 00:00:00 2001 From: asaharn Date: Tue, 6 May 2025 21:00:36 +0530 Subject: [PATCH 11/28] tests fixes --- .../outputs/microsoft_fabric/microsoft_fabric_test.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go index b66cb0330317f..9662754fb0bca 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go @@ -154,18 +154,12 @@ func TestMicrosoftFabric_Init(t *testing.T) { { name: "Empty connection string", connectionString: "", - expectedError: "endpoint must not be empty. For EventHouse refer : " + - "https://learn.microsoft.com/kusto/api/connection-strings/kusto?view=microsoft-fabric" + - " for EventStream refer : " + - "https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/add-manage-eventstream-sources?pivots=enhanced-capabilities", + expectedError: "endpoint must not be empty", }, { name: "Invalid connection string", connectionString: "invalid_connection_string", - expectedError: "invalid connection string. For EventHouse refer : " + - "https://learn.microsoft.com/kusto/api/connection-strings/kusto?view=microsoft-fabric" + - " for EventStream refer : " + - "https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/add-manage-eventstream-sources?pivots=enhanced-capabilities", + expectedError: "invalid connection string", }, { name: "Valid EventHouse connection string", From b0dc85c936f7f205fe6ed0139e5ca14b1d4516ed Mon Sep 17 00:00:00 2001 From: asaharn Date: Wed, 7 May 2025 10:15:57 +0530 Subject: [PATCH 12/28] formatting --- plugins/outputs/microsoft_fabric/microsoft_fabric_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go index 9662754fb0bca..67278d49ae13c 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go @@ -154,12 +154,12 @@ func TestMicrosoftFabric_Init(t *testing.T) { { name: "Empty connection string", connectionString: "", - expectedError: "endpoint must not be empty", + expectedError: "endpoint must not be empty", }, { name: "Invalid connection string", connectionString: "invalid_connection_string", - expectedError: "invalid connection string", + expectedError: "invalid connection string", }, { name: "Valid EventHouse connection string", From 9f721d7a09f6a9107827adb352f755c61acab0a9 Mon Sep 17 00:00:00 2001 From: asaharn Date: Wed, 7 May 2025 16:01:37 +0530 Subject: [PATCH 13/28] moved fabricoutput struct --- plugins/outputs/microsoft_fabric/fabric_output.go | 10 ---------- plugins/outputs/microsoft_fabric/microsoft_fabric.go | 9 ++++++++- 2 files changed, 8 insertions(+), 11 deletions(-) delete mode 100644 plugins/outputs/microsoft_fabric/fabric_output.go diff --git a/plugins/outputs/microsoft_fabric/fabric_output.go b/plugins/outputs/microsoft_fabric/fabric_output.go deleted file mode 100644 index d32daf3f0d865..0000000000000 --- a/plugins/outputs/microsoft_fabric/fabric_output.go +++ /dev/null @@ -1,10 +0,0 @@ -package microsoft_fabric - -import "github.com/influxdata/telegraf" - -type FabricOutput interface { - Init() error - Connect() error - Write(metrics []telegraf.Metric) error - Close() error -} diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric.go b/plugins/outputs/microsoft_fabric/microsoft_fabric.go index 0ac7d0f8fb2d8..58a117c2f95fe 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric.go @@ -17,13 +17,20 @@ import ( //go:embed sample.conf var sampleConfig string +type fabricOutput interface { + Init() error + Connect() error + Write(metrics []telegraf.Metric) error + Close() error +} + type MicrosoftFabric struct { ConnectionString string `toml:"connection_string"` Eventhouse *EventHouse `toml:"eventhouse"` Eventstream *EventStream `toml:"eventstream"` Log telegraf.Logger `toml:"-"` - activePlugin FabricOutput + activePlugin fabricOutput } func (*MicrosoftFabric) SampleConfig() string { From 808242a63c7a97ffa4ee666413ad572f8db6f964 Mon Sep 17 00:00:00 2001 From: asaharn Date: Mon, 19 May 2025 18:07:09 +0530 Subject: [PATCH 14/28] Changes to make connection string based property parsing --- .../microsoft_fabric/EVENTHOUSE_CONFIGS.md | 40 +------- plugins/outputs/microsoft_fabric/README.md | 96 +++++++------------ .../outputs/microsoft_fabric/event_house.go | 70 +++++++++++--- .../outputs/microsoft_fabric/event_stream.go | 70 ++++++++++---- .../microsoft_fabric/microsoft_fabric.go | 41 ++++---- .../microsoft_fabric/microsoft_fabric_test.go | 8 +- 6 files changed, 174 insertions(+), 151 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/EVENTHOUSE_CONFIGS.md b/plugins/outputs/microsoft_fabric/EVENTHOUSE_CONFIGS.md index a96b6153c6b63..691187040f165 100644 --- a/plugins/outputs/microsoft_fabric/EVENTHOUSE_CONFIGS.md +++ b/plugins/outputs/microsoft_fabric/EVENTHOUSE_CONFIGS.md @@ -1,41 +1,11 @@ # Eventhouse Configuration -```toml @sample.conf -[outputs.microsoft_fabric.eventstream] - ## The full connection string to the Event Hub (required) - ## The shared access key must have "Send" permissions on the target Event Hub. - - ## Client timeout - # timeout = "30s" - - ## Partition key - ## Metric tag or field name to use for the event partition key. The value of - ## this tag or field is set as the key for events if it exists. If both, tag - ## and field, exist the tag is preferred. - # partition_key = "" - - ## Set the maximum batch message size in bytes - ## The allowable size depends on the Event Hub tier; not setting this option or setting - ## it to zero will use the default size of the Azure Event Hubs Client library. See - ## https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas#basic-vs-standard-vs-premium-vs-dedicated-tiers - ## for the allowable size of your tier. - # max_message_size = "0B" - - ## Data format to output. - ## Each data format has its own unique set of configuration options, read - ## more about them here: - ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md - data_format = "json" - -``` - ## Metrics Grouping Metrics can be grouped in two ways to be sent to Azure Data Explorer. To specify which metric grouping type the plugin should use, the respective value should be -given to the `metrics_grouping_type` in the config file. If no value is given to -`metrics_grouping_type`, by default, the metrics will be grouped using -`TablePerMetric`. +given to the `Metrics Grouping Type` in the connection string. If no value is given, by default, the metrics will be grouped using +`tablepermetric`. ### TablePerMetric @@ -62,8 +32,8 @@ existing table. For more information about the merge process check the ## Tables Schema -The schema of the Azure Data Explorer table will match the structure of the -Telegraf `Metric` object. The corresponding Azure Data Explorer command +The schema of the Eventhouse table will match the structure of the +Telegraf `Metric` object. The corresponding command generated by the plugin would be like the following: ```text @@ -76,7 +46,7 @@ The corresponding table mapping would be like the following: .create-or-alter table ['table-name'] ingestion json mapping 'table-name_mapping' '[{"column":"fields", "Properties":{"Path":"$[\'fields\']"}},{"column":"name", "Properties":{"Path":"$[\'name\']"}},{"column":"tags", "Properties":{"Path":"$[\'tags\']"}},{"column":"timestamp", "Properties":{"Path":"$[\'timestamp\']"}}]' ``` -**Note**: This plugin will automatically create Azure Data Explorer tables and +**Note**: This plugin will automatically create Eventhouse tables and corresponding table mapping as per the above mentioned commands. ## Ingestion type diff --git a/plugins/outputs/microsoft_fabric/README.md b/plugins/outputs/microsoft_fabric/README.md index 705fbabc2bdc8..4fe9aed1dabf0 100644 --- a/plugins/outputs/microsoft_fabric/README.md +++ b/plugins/outputs/microsoft_fabric/README.md @@ -22,70 +22,19 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. ```toml # Sends metrics to Microsoft Fabric [[outputs.microsoft_fabric]] - ## The URI property of the Eventhouse resource on Azure - ## ex: connection_string = "Data Source=https://myadxresource.australiasoutheast.kusto.windows.net" - connection_string = "" - - - ## Using this section the plugin will send metrics to an Eventhouse endpoint - ## for ingesting, storing, and querying large volumes of data with low latency. - [outputs.microsoft_fabric.eventhouse] - [outputs.microsoft_fabric.eventhouse.cluster_config] - ## Database metrics will be written to - ## NOTE: The plugin will NOT generate the database. It is expected the database already exists. - database = "" - - ## Timeout for Eventhouse operations - # timeout = "20s" - - ## Type of metrics grouping; available options are: - ## tablepermetric -- for one table per distinct metric - ## singletable -- for writing all metrics to the same table - # metrics_grouping_type = "tablepermetric" - - # Name of the table to store metrics - ## NOTE: This option is only used for "singletable" metrics grouping - # table_name = "" - - ## Creates tables and relevant mapping - ## Disable when running with the lowest possible permissions i.e. table ingestor role. - # create_tables = true - - ## Ingestion method to use; available options are - ## - managed -- streaming ingestion with fallback to batched ingestion or the "queued" method below - ## - queued -- queue up metrics data and process sequentially - # ingestion_type = "queued" - - ## Using this section the plugin will send metrics to an EventStream endpoint - ## for transforming and routing metrics to various destinations without writing - ## any code. - [outputs.microsoft_fabric.eventstream] - ## The full connection string to the Event Hub (required) - ## The shared access key must have "Send" permissions on the target Event Hub. - - ## Client timeout - # timeout = "30s" - - ## Partition key - ## Metric tag or field name to use for the event partition key. The value of - ## this tag or field is set as the key for events if it exists. If both, tag - ## and field, exist the tag is preferred. - # partition_key = "" - - ## Set the maximum batch message size in bytes - ## The allowable size depends on the Event Hub tier; not setting this option or setting - ## it to zero will use the default size of the Azure Event Hubs Client library. See - ## https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas#basic-vs-standard-vs-premium-vs-dedicated-tiers - ## for the allowable size of your tier. - # max_message_size = "0B" - - ## Data format to output. - ## Each data format has its own unique set of configuration options, read - ## more about them here: - ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md - data_format = "json" + ## The URI property of the resource on Azure + connection_string = "https://trd-abcd.xx.kusto.fabric.microsoft.com;Database=kusto_eh; Table Name=telegraf_dump;Key=value" + + ## Client timeout + timeout = "30s" ``` +### Connection String + +Fabric telegraf output plugin connection string property provide the information necessary for a client application to establish a connection to a Fabric service endpoint. The connection string is a semicolon-delimited list of name-value parameter pairs, optionally prefixed by a single URI. + +Example: data source=; Table Name=telegraf_dump;Key=value + ### EventHouse The microsoft_fabric output plugin sends metrics to Microsoft Fabric, @@ -96,6 +45,20 @@ scalable data store designed for real-time analytics. It allows you to ingest, store, and query large volumes of data with low latency. For more information, visit the Eventhouse documentation. +The following table lists all the possible properties that can be included in a connection string and provide alias names for each property. + +### General properties + +| Property name | Description | +|---|---| +| Client Version for Tracing | The property used when tracing the client version. | +| Data Source

**Aliases:** Addr, Address, Network Address, Server | The URI specifying the Kusto service endpoint. For example, `https://mycluster.fabric.windows.net`. | +| Initial Catalog

**Alias:** Database | The default database name. For example, `MyDatabase`. | +| Ingestion Type

**Alias:** IngestionType | Values can be set to,
- managed : Streaming ingestion with fallback to batched ingestion or the "queued" method below
- queued : Queue up metrics data and process sequentially | +| Table Name

**Alias:** TableName | Name of the single table to store all the metrics (Only needed if metrics_grouping_type is "SingleTable") | +| Create Tables

**Alias:** CreateTables | Creates tables and relevant mapping if set to true(default).
Skips table and mapping creation if set to false, this is useful for running Telegraf with the lowest possible permissions i.e. table ingestor role. | +| Metrics Grouping Type

**Alias:** MetricsGroupingType | Type of metrics grouping used when pushing to Eventhouse. values can be set, 'tablepermetric' and 'singletable'. Default is "tablepermetric" for one table per different metric.| + More about the eventhouse configuration properties can be found [here](./EVENTHOUSE_CONFIGS.md) @@ -106,4 +69,13 @@ experience lets you bring real-time events into Fabric, transform them, and then route them to various destinations without writing any code (no-code). For more information, visit the [Eventstream documentation][eventstream_docs]. +To communicate with an eventstream, you need a connection string for the namespace or the event hub. If you use a connection string to the namespace from your application, following are the properties that can be added to the standard [Eventstream connection string][ecs] like a key value pair. + +| Property name | Description | +|---|---| +| Partition Key

**Aliases:** PartitionKey | Partition key to use for the event Metric tag or field name to use for the event partition key. The value of this tag or field is set as the key for events if it exists. If both, tag and field, exist the tag is preferred. | +| Max Message Size

**Aliases:** MaxMessageSize | Set the maximum batch message size in bytes The allowable size depends on the Event Hub tier, see for details. If unset the default size defined by Azure Event Hubs is used (currently 1,000,000 bytes) | + +[ecs]: https://learn.microsoft.com/azure/event-hubs/event-hubs-get-connection-string + [eventstream_docs]: https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/overview?tabs=enhancedcapabilities diff --git a/plugins/outputs/microsoft_fabric/event_house.go b/plugins/outputs/microsoft_fabric/event_house.go index 657add09ea54b..4b3ad6cee0141 100644 --- a/plugins/outputs/microsoft_fabric/event_house.go +++ b/plugins/outputs/microsoft_fabric/event_house.go @@ -2,6 +2,7 @@ package microsoft_fabric import ( "fmt" + "strings" "time" "github.com/Azure/azure-kusto-go/kusto/ingest" @@ -12,14 +13,15 @@ import ( "github.com/influxdata/telegraf/plugins/serializers/json" ) -type EventHouse struct { - Config *adx.Config `toml:"cluster_config"` - client *adx.Client +type eventhouse struct { + config *adx.Config + client *adx.Client + log telegraf.Logger serializer telegraf.Serializer } -func (e *EventHouse) Init() error { +func (e *eventhouse) Init() error { serializer := &json.Serializer{ TimestampUnits: config.Duration(time.Nanosecond), TimestampFormat: time.RFC3339Nano, @@ -28,11 +30,13 @@ func (e *EventHouse) Init() error { return err } e.serializer = serializer + e.config = &adx.Config{} + e.config.CreateTables = true return nil } -func (e *EventHouse) Connect() error { - client, err := e.Config.NewClient("Kusto.Telegraf", e.log) +func (e *eventhouse) Connect() error { + client, err := e.config.NewClient("Kusto.Telegraf", e.log) if err != nil { return fmt.Errorf("creating new client failed: %w", err) } @@ -41,18 +45,18 @@ func (e *EventHouse) Connect() error { return nil } -func (e *EventHouse) Write(metrics []telegraf.Metric) error { - if e.Config.MetricsGrouping == adx.TablePerMetric { +func (e *eventhouse) Write(metrics []telegraf.Metric) error { + if e.config.MetricsGrouping == adx.TablePerMetric { return e.writeTablePerMetric(metrics) } return e.writeSingleTable(metrics) } -func (e *EventHouse) Close() error { +func (e *eventhouse) Close() error { return e.client.Close() } -func (e *EventHouse) writeTablePerMetric(metrics []telegraf.Metric) error { +func (e *eventhouse) writeTablePerMetric(metrics []telegraf.Metric) error { tableMetricGroups := make(map[string][]byte) // Group metrics by name and serialize them for _, m := range metrics { @@ -79,7 +83,7 @@ func (e *EventHouse) writeTablePerMetric(metrics []telegraf.Metric) error { return nil } -func (e *EventHouse) writeSingleTable(metrics []telegraf.Metric) error { +func (e *eventhouse) writeSingleTable(metrics []telegraf.Metric) error { // serialise each metric in metrics - store in byte[] metricsArray := make([]byte, 0) for _, m := range metrics { @@ -92,6 +96,48 @@ func (e *EventHouse) writeSingleTable(metrics []telegraf.Metric) error { // push metrics to a single table format := ingest.FileFormat(ingest.JSON) - err := e.client.PushMetrics(format, e.Config.TableName, metricsArray) + err := e.client.PushMetrics(format, e.config.TableName, metricsArray) return err } + +func (e *eventhouse) parseconnectionString(cs string) error { + // Parse the connection string to extract the endpoint and database + if cs == "" { + return fmt.Errorf("connection string must not be empty") + } + // Split the connection string into key-value pairs + pairs := strings.Split(cs, ";") + for _, pair := range pairs { + // Split each pair into key and value + k, v, found := strings.Cut(pair, "=") + if !found { + return fmt.Errorf("invalid connection string format: %s", pair) + } + k = strings.ToLower(strings.TrimSpace(k)) + v = strings.TrimSpace(v) + switch k { + case "data source", "addr", "address", "network address", "server": + e.config.Endpoint = v + case "initial catalog", "database": + e.config.Database = v + case "ingestion type", "ingestiontype": + e.config.IngestionType = v + case "table name", "tablename": + e.config.TableName = v + case "create tables", "createtables": + if v == "false" { + e.config.CreateTables = false + } else { + e.config.CreateTables = true + } + case "metrics grouping type, metricsgroupingtype": + if v == adx.TablePerMetric || v == adx.SingleTable { + fmt.Printf("Setting metrics grouping type to %q\n", v) + e.config.MetricsGrouping = v + } else { + return fmt.Errorf("invalid metrics grouping type: %s", v) + } + } + } + return nil +} diff --git a/plugins/outputs/microsoft_fabric/event_stream.go b/plugins/outputs/microsoft_fabric/event_stream.go index 9aac264894f59..245681fe0221b 100644 --- a/plugins/outputs/microsoft_fabric/event_stream.go +++ b/plugins/outputs/microsoft_fabric/event_stream.go @@ -5,6 +5,9 @@ import ( "context" "errors" "fmt" + "slices" + "strconv" + "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" @@ -15,10 +18,10 @@ import ( "github.com/influxdata/telegraf/plugins/serializers/json" ) -type EventStream struct { - PartitionKey string `toml:"partition_key"` - MaxMessageSize config.Size `toml:"max_message_size"` - Timeout config.Duration `toml:"timeout"` +type eventstream struct { + partitionKey string + maxMessageSize config.Size + timeout config.Duration connectionString string log telegraf.Logger @@ -27,7 +30,9 @@ type EventStream struct { serializer telegraf.Serializer } -func (e *EventStream) Init() error { +var confKeys = []string{"PartitionKey", "MaxMessageSize"} + +func (e *eventstream) Init() error { serializer := &json.Serializer{ TimestampUnits: config.Duration(time.Nanosecond), TimestampFormat: time.RFC3339Nano, @@ -36,14 +41,13 @@ func (e *EventStream) Init() error { return err } e.serializer = serializer - if e.MaxMessageSize > 0 { - e.options.MaxBytes = uint64(e.MaxMessageSize) + if e.maxMessageSize > 0 { + e.options.MaxBytes = uint64(e.maxMessageSize) } - return nil } -func (e *EventStream) Connect() error { +func (e *eventstream) Connect() error { cfg := &azeventhubs.ProducerClientOptions{ ApplicationID: internal.FormatFullVersion(), RetryOptions: azeventhubs.RetryOptions{MaxRetries: -1}, @@ -58,18 +62,18 @@ func (e *EventStream) Connect() error { return nil } -func (e *EventStream) Close() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(e.Timeout)) +func (e *eventstream) Close() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(e.timeout)) defer cancel() return e.client.Close(ctx) } -func (e *EventStream) SetSerializer(serializer telegraf.Serializer) { +func (e *eventstream) SetSerializer(serializer telegraf.Serializer) { e.serializer = serializer } -func (e *EventStream) Write(metrics []telegraf.Metric) error { +func (e *eventstream) Write(metrics []telegraf.Metric) error { // This context is only used for creating the batches which should not timeout as this is // not an I/O operation. Therefore avoid setting a timeout here. ctx := context.Background() @@ -90,10 +94,10 @@ func (e *EventStream) Write(metrics []telegraf.Metric) error { // Get the batcher for the chosen partition partition := "" - if e.PartitionKey != "" { - if key, ok := m.GetTag(e.PartitionKey); ok { + if e.partitionKey != "" { + if key, ok := m.GetTag(e.partitionKey); ok { partition = key - } else if key, ok := m.GetField(e.PartitionKey); ok { + } else if key, ok := m.GetField(e.partitionKey); ok { if k, ok := key.(string); ok { partition = k } @@ -150,8 +154,38 @@ func (e *EventStream) Write(metrics []telegraf.Metric) error { return nil } -func (e *EventStream) send(ctx context.Context, batch *azeventhubs.EventDataBatch) error { - ctx, cancel := context.WithTimeout(ctx, time.Duration(e.Timeout)) +func (e *eventstream) parseconnectionString(cs string) error { + // Parse the connection string + if cs == "" { + return fmt.Errorf("connection string must not be empty") + } + // Split the connection string into key-value pairs + pairs := strings.Split(cs, ";") + for _, pair := range pairs { + // Split each pair into key and value + k, v, found := strings.Cut(pair, "=") + if !found { + return fmt.Errorf("invalid connection string format: %s", pair) + } + k = strings.ToLower(strings.TrimSpace(k)) + v = strings.TrimSpace(v) + if slices.Contains(confKeys, k) { + switch k { + case "partitionkey", "partition key": + e.partitionKey = v + case "maxmessagesize", "max message size": + if sz, err := strconv.ParseInt(v, 10, 64); err != nil { + e.maxMessageSize = config.Size(sz) + } + } + } + } + return nil + +} + +func (e *eventstream) send(ctx context.Context, batch *azeventhubs.EventDataBatch) error { + ctx, cancel := context.WithTimeout(ctx, time.Duration(e.timeout)) defer cancel() return e.client.SendEventDataBatch(ctx, batch, nil) diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric.go b/plugins/outputs/microsoft_fabric/microsoft_fabric.go index 58a117c2f95fe..2d3d53dad5e1b 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric.go @@ -10,7 +10,6 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/config" - "github.com/influxdata/telegraf/plugins/common/adx" "github.com/influxdata/telegraf/plugins/outputs" ) @@ -26,10 +25,11 @@ type fabricOutput interface { type MicrosoftFabric struct { ConnectionString string `toml:"connection_string"` - Eventhouse *EventHouse `toml:"eventhouse"` - Eventstream *EventStream `toml:"eventstream"` Log telegraf.Logger `toml:"-"` + Timeout config.Duration `toml:"timeout"` + eventhouse *eventhouse + eventstream *eventstream activePlugin fabricOutput } @@ -46,22 +46,30 @@ func (m *MicrosoftFabric) Init() error { if strings.HasPrefix(connectionString, "Endpoint=sb") { m.Log.Info("Detected EventStream endpoint, using EventStream output plugin") - - m.Eventstream.connectionString = connectionString - m.Eventstream.log = m.Log - if err := m.Eventstream.Init(); err != nil { + eventstream := &eventstream{} + eventstream.connectionString = connectionString + eventstream.log = m.Log + eventstream.timeout = m.Timeout + eventstream.parseconnectionString(connectionString) + m.eventstream = eventstream + if err := m.eventstream.Init(); err != nil { return fmt.Errorf("initializing EventStream output failed: %w", err) } - m.activePlugin = m.Eventstream + m.activePlugin = eventstream } else if isKustoEndpoint(strings.ToLower(connectionString)) { m.Log.Info("Detected EventHouse endpoint, using EventHouse output plugin") // Setting up the AzureDataExplorer plugin initial properties - m.Eventhouse.Config.Endpoint = connectionString - m.Eventhouse.log = m.Log - if err := m.Eventhouse.Init(); err != nil { + eventhouse := &eventhouse{} + m.eventhouse = eventhouse + if err := m.eventhouse.Init(); err != nil { return fmt.Errorf("initializing EventHouse output failed: %w", err) } - m.activePlugin = m.Eventhouse + eventhouse.config.Endpoint = connectionString + eventhouse.log = m.Log + eventhouse.config.Timeout = m.Timeout + eventhouse.parseconnectionString(connectionString) + m.activePlugin = m.eventhouse + fmt.Printf("EventHouse plugin initialized successfully %#v\n ------- %#v\n", eventhouse.client, eventhouse.config) } else { return errors.New("invalid connection string") } @@ -109,14 +117,7 @@ func isKustoEndpoint(endpoint string) bool { func init() { outputs.Add("microsoft_fabric", func() telegraf.Output { return &MicrosoftFabric{ - Eventstream: &EventStream{ - Timeout: config.Duration(30 * time.Second), - }, - Eventhouse: &EventHouse{ - Config: &adx.Config{ - Timeout: config.Duration(30 * time.Second), - }, - }, + Timeout: config.Duration(30 * time.Second), } }) } diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go index 67278d49ae13c..6a036dfb7fa3f 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go @@ -179,13 +179,13 @@ func TestMicrosoftFabric_Init(t *testing.T) { mf := &MicrosoftFabric{ ConnectionString: tt.connectionString, Log: testutil.Logger{}, - Eventhouse: &EventHouse{ - Config: &adx.Config{ + eventhouse: &eventhouse{ + config: &adx.Config{ Database: "database", }, }, - Eventstream: &EventStream{ - Timeout: config.Duration(30 * time.Second), + eventstream: &eventstream{ + timeout: config.Duration(30 * time.Second), }, } err := mf.Init() From 2deec8c7b003cb7d7abf269bf1fe29b707ec44ad Mon Sep 17 00:00:00 2001 From: asaharn Date: Tue, 20 May 2025 11:36:44 +0530 Subject: [PATCH 15/28] linting changes --- plugins/outputs/microsoft_fabric/event_house.go | 11 +++++------ plugins/outputs/microsoft_fabric/event_stream.go | 3 +-- plugins/outputs/microsoft_fabric/microsoft_fabric.go | 9 ++++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/event_house.go b/plugins/outputs/microsoft_fabric/event_house.go index 4b3ad6cee0141..489286587d06e 100644 --- a/plugins/outputs/microsoft_fabric/event_house.go +++ b/plugins/outputs/microsoft_fabric/event_house.go @@ -1,6 +1,7 @@ package microsoft_fabric import ( + "errors" "fmt" "strings" "time" @@ -103,7 +104,7 @@ func (e *eventhouse) writeSingleTable(metrics []telegraf.Metric) error { func (e *eventhouse) parseconnectionString(cs string) error { // Parse the connection string to extract the endpoint and database if cs == "" { - return fmt.Errorf("connection string must not be empty") + return errors.New("connection string must not be empty") } // Split the connection string into key-value pairs pairs := strings.Split(cs, ";") @@ -131,12 +132,10 @@ func (e *eventhouse) parseconnectionString(cs string) error { e.config.CreateTables = true } case "metrics grouping type, metricsgroupingtype": - if v == adx.TablePerMetric || v == adx.SingleTable { - fmt.Printf("Setting metrics grouping type to %q\n", v) - e.config.MetricsGrouping = v - } else { - return fmt.Errorf("invalid metrics grouping type: %s", v) + if v != adx.TablePerMetric && v != adx.SingleTable { + return errors.New("metrics grouping type is not valid:" + v) } + e.config.MetricsGrouping = v } } return nil diff --git a/plugins/outputs/microsoft_fabric/event_stream.go b/plugins/outputs/microsoft_fabric/event_stream.go index 245681fe0221b..2281bfdd82717 100644 --- a/plugins/outputs/microsoft_fabric/event_stream.go +++ b/plugins/outputs/microsoft_fabric/event_stream.go @@ -157,7 +157,7 @@ func (e *eventstream) Write(metrics []telegraf.Metric) error { func (e *eventstream) parseconnectionString(cs string) error { // Parse the connection string if cs == "" { - return fmt.Errorf("connection string must not be empty") + return errors.New("connection string must not be empty") } // Split the connection string into key-value pairs pairs := strings.Split(cs, ";") @@ -181,7 +181,6 @@ func (e *eventstream) parseconnectionString(cs string) error { } } return nil - } func (e *eventstream) send(ctx context.Context, batch *azeventhubs.EventDataBatch) error { diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric.go b/plugins/outputs/microsoft_fabric/microsoft_fabric.go index 2d3d53dad5e1b..0d3d1a3407046 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric.go @@ -50,7 +50,9 @@ func (m *MicrosoftFabric) Init() error { eventstream.connectionString = connectionString eventstream.log = m.Log eventstream.timeout = m.Timeout - eventstream.parseconnectionString(connectionString) + if err := eventstream.parseconnectionString(connectionString); err != nil { + return fmt.Errorf("parsing connection string failed: %w", err) + } m.eventstream = eventstream if err := m.eventstream.Init(); err != nil { return fmt.Errorf("initializing EventStream output failed: %w", err) @@ -67,9 +69,10 @@ func (m *MicrosoftFabric) Init() error { eventhouse.config.Endpoint = connectionString eventhouse.log = m.Log eventhouse.config.Timeout = m.Timeout - eventhouse.parseconnectionString(connectionString) + if err := eventhouse.parseconnectionString(connectionString); err != nil { + return fmt.Errorf("parsing connection string failed: %w", err) + } m.activePlugin = m.eventhouse - fmt.Printf("EventHouse plugin initialized successfully %#v\n ------- %#v\n", eventhouse.client, eventhouse.config) } else { return errors.New("invalid connection string") } From 67e8ae1665d105d15e8c464abd61042295c9c291 Mon Sep 17 00:00:00 2001 From: asaharn Date: Tue, 20 May 2025 12:08:21 +0530 Subject: [PATCH 16/28] Readme linting --- plugins/outputs/microsoft_fabric/README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/README.md b/plugins/outputs/microsoft_fabric/README.md index 4fe9aed1dabf0..f979459c31043 100644 --- a/plugins/outputs/microsoft_fabric/README.md +++ b/plugins/outputs/microsoft_fabric/README.md @@ -31,9 +31,13 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. ### Connection String -Fabric telegraf output plugin connection string property provide the information necessary for a client application to establish a connection to a Fabric service endpoint. The connection string is a semicolon-delimited list of name-value parameter pairs, optionally prefixed by a single URI. +Fabric telegraf output plugin connection string property provide the +information necessary for a client application to establish a connection + to a Fabric service endpoint. The connection string is a semicolon-delimited + list of name-value parameter pairs, optionally prefixed by a single URI. -Example: data source=; Table Name=telegraf_dump;Key=value +Example: data source=; +Table Name=telegraf_dump;Key=value ### EventHouse @@ -69,7 +73,10 @@ experience lets you bring real-time events into Fabric, transform them, and then route them to various destinations without writing any code (no-code). For more information, visit the [Eventstream documentation][eventstream_docs]. -To communicate with an eventstream, you need a connection string for the namespace or the event hub. If you use a connection string to the namespace from your application, following are the properties that can be added to the standard [Eventstream connection string][ecs] like a key value pair. +To communicate with an eventstream, you need a connection string for the +namespace or the event hub. If you use a connection string to the namespace +from your application, following are the properties that can be added +to the standard [Eventstream connection string][ecs] like a key value pair. | Property name | Description | |---|---| From ef49ef63931328228591371c2be656a74930c500 Mon Sep 17 00:00:00 2001 From: asaharn Date: Tue, 20 May 2025 12:10:49 +0530 Subject: [PATCH 17/28] readme linting --- plugins/outputs/microsoft_fabric/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/README.md b/plugins/outputs/microsoft_fabric/README.md index f979459c31043..9cc35399446be 100644 --- a/plugins/outputs/microsoft_fabric/README.md +++ b/plugins/outputs/microsoft_fabric/README.md @@ -36,8 +36,8 @@ information necessary for a client application to establish a connection to a Fabric service endpoint. The connection string is a semicolon-delimited list of name-value parameter pairs, optionally prefixed by a single URI. -Example: data source=; -Table Name=telegraf_dump;Key=value +Example: data source=;Table Name=telegraf_dump;Key=value ### EventHouse @@ -49,7 +49,8 @@ scalable data store designed for real-time analytics. It allows you to ingest, store, and query large volumes of data with low latency. For more information, visit the Eventhouse documentation. -The following table lists all the possible properties that can be included in a connection string and provide alias names for each property. +The following table lists all the possible properties that can be included in a +connection string and provide alias names for each property. ### General properties From 17c1ff8df9dfe5c869a203370c7bfc2d1d044be5 Mon Sep 17 00:00:00 2001 From: asaharn Date: Tue, 3 Jun 2025 00:41:43 +0530 Subject: [PATCH 18/28] *Added new test * Changes for review comments --- .../microsoft_fabric/EVENTHOUSE_CONFIGS.md | 231 ---------------- plugins/outputs/microsoft_fabric/README.md | 93 +++++-- .../outputs/microsoft_fabric/event_house.go | 59 ++-- .../microsoft_fabric/event_house_test.go | 234 ++++++++++++++++ .../outputs/microsoft_fabric/event_stream.go | 37 ++- .../microsoft_fabric/event_stream_test.go | 108 ++++++++ .../microsoft_fabric/microsoft_fabric.go | 48 +--- .../microsoft_fabric/microsoft_fabric_test.go | 254 +++++++----------- plugins/outputs/microsoft_fabric/sample.conf | 65 +---- 9 files changed, 588 insertions(+), 541 deletions(-) delete mode 100644 plugins/outputs/microsoft_fabric/EVENTHOUSE_CONFIGS.md create mode 100644 plugins/outputs/microsoft_fabric/event_house_test.go create mode 100644 plugins/outputs/microsoft_fabric/event_stream_test.go diff --git a/plugins/outputs/microsoft_fabric/EVENTHOUSE_CONFIGS.md b/plugins/outputs/microsoft_fabric/EVENTHOUSE_CONFIGS.md deleted file mode 100644 index 691187040f165..0000000000000 --- a/plugins/outputs/microsoft_fabric/EVENTHOUSE_CONFIGS.md +++ /dev/null @@ -1,231 +0,0 @@ -# Eventhouse Configuration - -## Metrics Grouping - -Metrics can be grouped in two ways to be sent to Azure Data Explorer. To specify -which metric grouping type the plugin should use, the respective value should be -given to the `Metrics Grouping Type` in the connection string. If no value is given, by default, the metrics will be grouped using -`tablepermetric`. - -### TablePerMetric - -The plugin will group the metrics by the metric name, and will send each group -of metrics to an Azure Data Explorer table. If the table doesn't exist the -plugin will create the table, if the table exists then the plugin will try to -merge the Telegraf metric schema to the existing table. For more information -about the merge process check the [`.create-merge` documentation][create-merge]. - -The table name will match the `name` property of the metric, this means that the -name of the metric should comply with the Azure Data Explorer table naming -constraints in case you plan to add a prefix to the metric name. - -[create-merge]: https://docs.microsoft.com/en-us/azure/data-explorer/kusto/management/create-merge-table-command - -### SingleTable - -The plugin will send all the metrics received to a single Azure Data Explorer -table. The name of the table must be supplied via `table_name` in the config -file. If the table doesn't exist the plugin will create the table, if the table -exists then the plugin will try to merge the Telegraf metric schema to the -existing table. For more information about the merge process check the -[`.create-merge` documentation][create-merge]. - -## Tables Schema - -The schema of the Eventhouse table will match the structure of the -Telegraf `Metric` object. The corresponding command -generated by the plugin would be like the following: - -```text -.create-merge table ['table-name'] (['fields']:dynamic, ['name']:string, ['tags']:dynamic, ['timestamp']:datetime) -``` - -The corresponding table mapping would be like the following: - -```text -.create-or-alter table ['table-name'] ingestion json mapping 'table-name_mapping' '[{"column":"fields", "Properties":{"Path":"$[\'fields\']"}},{"column":"name", "Properties":{"Path":"$[\'name\']"}},{"column":"tags", "Properties":{"Path":"$[\'tags\']"}},{"column":"timestamp", "Properties":{"Path":"$[\'timestamp\']"}}]' -``` - -**Note**: This plugin will automatically create Eventhouse tables and -corresponding table mapping as per the above mentioned commands. - -## Ingestion type - -**Note**: -[Streaming ingestion](https://aka.ms/AAhlg6s) -has to be enabled on ADX [configure the ADX cluster] -in case of `managed` option. -Refer the query below to check if streaming is enabled - -```kql -.show database policy streamingingestion -``` - -## Authentication - -### Supported Authentication Methods - -This plugin provides several types of authentication. The plugin will check the -existence of several specific environment variables, and consequently will -choose the right method. - -These methods are: - -1. AAD Application Tokens (Service Principals with secrets or certificates). - - For guidance on how to create and register an App in Azure Active Directory - check [this article][register], and for more information on the Service - Principals check [this article][principal]. - -2. AAD User Tokens - - - Allows Telegraf to authenticate like a user. This method is mainly used - for development purposes only. - -3. Managed Service Identity (MSI) token - - - If you are running Telegraf from Azure VM or infrastructure, then this is - the preferred authentication method. - -[register]: https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app#register-an-application - -[principal]: https://docs.microsoft.com/en-us/azure/active-directory/develop/app-objects-and-service-principals - -Whichever method, the designated Principal needs to be assigned the `Database -User` role on the Database level in the Azure Data Explorer. This role will -allow the plugin to create the required tables and ingest data into it. If -`create_tables=false` then the designated principal only needs the `Database -Ingestor` role at least. - -### Configurations of the chosen Authentication Method - -The plugin will authenticate using the first available of the following -configurations, **it's important to understand that the assessment, and -consequently choosing the authentication method, will happen in order as -below**: - -1. **Client Credentials**: Azure AD Application ID and Secret. - - Set the following environment variables: - - - `AZURE_TENANT_ID`: Specifies the Tenant to which to authenticate. - - `AZURE_CLIENT_ID`: Specifies the app client ID to use. - - `AZURE_CLIENT_SECRET`: Specifies the app secret to use. - -2. **Client Certificate**: Azure AD Application ID and X.509 Certificate. - - - `AZURE_TENANT_ID`: Specifies the Tenant to which to authenticate. - - `AZURE_CLIENT_ID`: Specifies the app client ID to use. - - `AZURE_CERTIFICATE_PATH`: Specifies the certificate Path to use. - - `AZURE_CERTIFICATE_PASSWORD`: Specifies the certificate password to use. - -3. **Resource Owner Password**: Azure AD User and Password. This grant type is - *not recommended*, use device login instead if you need interactive login. - - - `AZURE_TENANT_ID`: Specifies the Tenant to which to authenticate. - - `AZURE_CLIENT_ID`: Specifies the app client ID to use. - - `AZURE_USERNAME`: Specifies the username to use. - - `AZURE_PASSWORD`: Specifies the password to use. - -4. **Azure Managed Service Identity**: Delegate credential management to the - platform. Requires that code is running in Azure, e.g. on a VM. All - configuration is handled by Azure. See [Azure Managed Service Identity][msi] - for more details. Only available when using the [Azure Resource - Manager][arm]. - -[msi]: https://docs.microsoft.com/en-us/azure/active-directory/msi-overview -[arm]: https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-overview - -## Querying data collected in Azure Data Explorer - -Examples of data transformations and queries that would be useful to gain -insights - - -### Using SQL input plugin - -Sample SQL metrics data - - -name | tags | timestamp | fields ------|------|-----------|------- -sqlserver_database_io|{"database_name":"azure-sql-db2","file_type":"DATA","host":"adx-vm","logical_filename":"tempdev","measurement_db_type":"AzureSQLDB","physical_filename":"tempdb.mdf","replica_updateability":"READ_WRITE","sql_instance":"adx-sql-server"}|2021-09-09T13:51:20Z|{"current_size_mb":16,"database_id":2,"file_id":1,"read_bytes":2965504,"read_latency_ms":68,"reads":47,"rg_read_stall_ms":42,"rg_write_stall_ms":0,"space_used_mb":0,"write_bytes":1220608,"write_latency_ms":103,"writes":149} -sqlserver_waitstats|{"database_name":"azure-sql-db2","host":"adx-vm","measurement_db_type":"AzureSQLDB","replica_updateability":"READ_WRITE","sql_instance":"adx-sql-server","wait_category":"Worker Thread","wait_type":"THREADPOOL"}|2021-09-09T13:51:20Z|{"max_wait_time_ms":15,"resource_wait_ms":4469,"signal_wait_time_ms":0,"wait_time_ms":4469,"waiting_tasks_count":1464} - -Since collected metrics object is of complex type so "fields" and "tags" are -stored as dynamic data type, multiple ways to query this data- - -1. Query JSON attributes directly: Azure Data Explorer provides an ability to - query JSON data in raw format without parsing it, so JSON attributes can be - queried directly in following way: - - ```text - Tablename - | where name == "sqlserver_azure_db_resource_stats" and todouble(fields.avg_cpu_percent) > 7 - ``` - - ```text - Tablename - | distinct tostring(tags.database_name) - ``` - - **Note** - This approach could have performance impact in case of large - volumes of data, use below mentioned approach for such cases. - -1. Use [Update - policy](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/management/updatepolicy)**: - Transform dynamic data type columns using update policy. This is the - recommended performant way for querying over large volumes of data compared - to querying directly over JSON attributes: - - ```json - // Function to transform data - .create-or-alter function Transform_TargetTableName() { - SourceTableName - | mv-apply fields on (extend key = tostring(bag_keys(fields)[0])) - | project fieldname=key, value=todouble(fields[key]), name, tags, timestamp - } - - // Create destination table with above query's results schema (if it doesn't exist already) - .set-or-append TargetTableName <| Transform_TargetTableName() | limit 0 - - // Apply update policy on destination table - .alter table TargetTableName policy update - @'[{"IsEnabled": true, "Source": "SourceTableName", "Query": "Transform_TargetTableName()", "IsTransactional": true, "PropagateIngestionProperties": false}]' - ``` - -### Using syslog input plugin - -Sample syslog data - - -name | tags | timestamp | fields ------|------|-----------|------- -syslog|{"appname":"azsecmond","facility":"user","host":"adx-linux-vm","hostname":"adx-linux-vm","severity":"info"}|2021-09-20T14:36:44Z|{"facility_code":1,"message":" 2021/09/20 14:36:44.890110 Failed to connect to mdsd: dial unix /var/run/mdsd/default_djson.socket: connect: no such file or directory","procid":"2184","severity_code":6,"timestamp":"1632148604890477000","version":1} -syslog|{"appname":"CRON","facility":"authpriv","host":"adx-linux-vm","hostname":"adx-linux-vm","severity":"info"}|2021-09-20T14:37:01Z|{"facility_code":10,"message":" pam_unix(cron:session): session opened for user root by (uid=0)","procid":"26446","severity_code":6,"timestamp":"1632148621120781000","version":1} - -There are multiple ways to flatten dynamic columns using 'extend' or -'bag_unpack' operator. You can use either of these ways in above mentioned -update policy function - 'Transform_TargetTableName()' - -- Use - [extend](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/extendoperator) - operator - This is the recommended approach compared to 'bag_unpack' as it is - faster and robust. Even if schema changes, it will not break queries or - dashboards. - - ```text - Tablenmae - | extend facility_code=toint(fields.facility_code), message=tostring(fields.message), procid= tolong(fields.procid), severity_code=toint(fields.severity_code), - SysLogTimestamp=unixtime_nanoseconds_todatetime(tolong(fields.timestamp)), version= todouble(fields.version), - appname= tostring(tags.appname), facility= tostring(tags.facility),host= tostring(tags.host), hostname=tostring(tags.hostname), severity=tostring(tags.severity) - | project-away fields, tags - ``` - -- Use [bag_unpack - plugin](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/bag-unpackplugin) - to unpack the dynamic type columns automatically. This method could lead to - issues if source schema changes as its dynamically expanding columns. - - ```text - Tablename - | evaluate bag_unpack(tags, columnsConflict='replace_source') - | evaluate bag_unpack(fields, columnsConflict='replace_source') - ``` diff --git a/plugins/outputs/microsoft_fabric/README.md b/plugins/outputs/microsoft_fabric/README.md index 9cc35399446be..e940cd70eccde 100644 --- a/plugins/outputs/microsoft_fabric/README.md +++ b/plugins/outputs/microsoft_fabric/README.md @@ -31,28 +31,28 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. ### Connection String -Fabric telegraf output plugin connection string property provide the -information necessary for a client application to establish a connection - to a Fabric service endpoint. The connection string is a semicolon-delimited - list of name-value parameter pairs, optionally prefixed by a single URI. - -Example: data source=;Table Name=telegraf_dump;Key=value +The `connection_string` provide the information necessary for a client application to +establish a connection to the Fabric service endpoint. It is a +semicolon-delimited list of name-value parameter pairs, optionally prefixed by a +single URI. The `connection_string` setting is specific to the type of endpoint you are using. +The sections below will detail on the required and available name-value pairs for +each type. ### EventHouse -The microsoft_fabric output plugin sends metrics to Microsoft Fabric, -a scalable data platform for real-time analytics. This plugin allows you to leverage Microsoft Fabric's capabilities to store and analyze your Telegraf metrics. Eventhouse is a high-performance, scalable data store designed for real-time analytics. It allows you to ingest, store, and query large volumes of data with low latency. For more information, -visit the Eventhouse documentation. +visit the [Eventhouse documentation][eventhousedocs]. + +[eventhousedocs]: https://learn.microsoft.com/fabric/real-time-intelligence/eventhouse The following table lists all the possible properties that can be included in a -connection string and provide alias names for each property. +connection string and provide alias names for each property: -### General properties +The following table lists all the possible properties that can be included in a +connection string and provide alias names for each property. | Property name | Description | |---|---| @@ -64,13 +64,74 @@ connection string and provide alias names for each property. | Create Tables

**Alias:** CreateTables | Creates tables and relevant mapping if set to true(default).
Skips table and mapping creation if set to false, this is useful for running Telegraf with the lowest possible permissions i.e. table ingestor role. | | Metrics Grouping Type

**Alias:** MetricsGroupingType | Type of metrics grouping used when pushing to Eventhouse. values can be set, 'tablepermetric' and 'singletable'. Default is "tablepermetric" for one table per different metric.| -More about the eventhouse configuration properties -can be found [here](./EVENTHOUSE_CONFIGS.md) +* *Metrics Grouping* + + Metrics can be grouped in two ways to be sent to Azure Data Explorer. To specify + which metric grouping type the plugin should use, the respective value should be + given to the `Metrics Grouping Type` in the connection string. If no value is given, by default, the metrics will be grouped using + `tablepermetric`. + +* *TablePerMetric* + + The plugin will group the metrics by the metric name, and will send each group + of metrics to an Azure Data Explorer table. If the table doesn't exist the + plugin will create the table, if the table exists then the plugin will try to + merge the Telegraf metric schema to the existing table. For more information +about the merge process check the [`.create-merge` documentation][create-merge]. + + The table name will match the `name` property of the metric, this means that the + name of the metric should comply with the Azure Data Explorer table naming + constraints in case you plan to add a prefix to the metric name. + +[create-merge]: https://docs.microsoft.com/en-us/azure/data-explorer/kusto/management/create-merge-table-command + +* *SingleTable* + + The plugin will send all the metrics received to a single Azure Data Explorer + table. The name of the table must be supplied via `table_name` in the config + file. If the table doesn't exist the plugin will create the table, if the table + exists then the plugin will try to merge the Telegraf metric schema to the + existing table. For more information about the merge process check the + [`.create-merge` documentation][create-merge]. + +* *Tables Schema* + + The schema of the Eventhouse table will match the structure of the + Telegraf `Metric` object. The corresponding command + generated by the plugin would be like the following: + + ```kql + .create-merge table ['table-name'] (['fields']:dynamic, ['name']:string, ['tags']:dynamic, ['timestamp']:datetime) + ``` + + The corresponding table mapping would be like the following: + + ```kql + .create-or-alter table ['table-name'] ingestion json mapping 'table-name_mapping' '[{"column":"fields", "Properties":{"Path":"$[\'fields\']"}},{"column":"name", "Properties":{"Path":"$[\'name\']"}},{"column":"tags", "Properties":{"Path":"$[\'tags\']"}},{"column":"timestamp", "Properties":{"Path":"$[\'timestamp\']"}}]' + ``` + + **Note**: This plugin will automatically create Eventhouse tables and + corresponding table mapping as per the above mentioned commands. + +* *Ingestion type* + + **Note**: + [Streaming ingestion](https://aka.ms/AAhlg6s) + has to be enabled on ADX [configure the ADX cluster] + in case of `managed` option. + Refer the query below to check if streaming is enabled + + ```kql + .show database policy streamingingestion + ``` + + To know more about configuration, supported authentication methods and querying ingested data, read the [documentation][ethdocs] + + [ethdocs]: https://learn.microsoft.com/azure/data-explorer/ingest-data-telegraf ### Eventstream -The eventstreams feature in the Microsoft Fabric Real-Time Intelligence -experience lets you bring real-time events into Fabric, transform them, +Eventstreams allow you to bring real-time events into Fabric, transform them, and then route them to various destinations without writing any code (no-code). For more information, visit the [Eventstream documentation][eventstream_docs]. diff --git a/plugins/outputs/microsoft_fabric/event_house.go b/plugins/outputs/microsoft_fabric/event_house.go index 489286587d06e..c6702e9e1e060 100644 --- a/plugins/outputs/microsoft_fabric/event_house.go +++ b/plugins/outputs/microsoft_fabric/event_house.go @@ -15,9 +15,9 @@ import ( ) type eventhouse struct { - config *adx.Config - client *adx.Client + adx.Config + client *adx.Client log telegraf.Logger serializer telegraf.Serializer } @@ -28,16 +28,33 @@ func (e *eventhouse) Init() error { TimestampFormat: time.RFC3339Nano, } if err := serializer.Init(); err != nil { - return err + return fmt.Errorf("initializing JSON serializer failed: %w", err) } e.serializer = serializer - e.config = &adx.Config{} - e.config.CreateTables = true + e.Config = adx.Config{} + e.CreateTables = true return nil } +func isEventhouseEndpoint(endpoint string) bool { + prefixes := []string{ + "data source=", + "addr=", + "address=", + "network address=", + "server=", + } + + for _, prefix := range prefixes { + if strings.HasPrefix(strings.ToLower(endpoint), prefix) { + return true + } + } + return false +} + func (e *eventhouse) Connect() error { - client, err := e.config.NewClient("Kusto.Telegraf", e.log) + client, err := e.NewClient("Kusto.Telegraf", e.log) if err != nil { return fmt.Errorf("creating new client failed: %w", err) } @@ -47,7 +64,7 @@ func (e *eventhouse) Connect() error { } func (e *eventhouse) Write(metrics []telegraf.Metric) error { - if e.config.MetricsGrouping == adx.TablePerMetric { + if e.MetricsGrouping == adx.TablePerMetric { return e.writeTablePerMetric(metrics) } return e.writeSingleTable(metrics) @@ -97,15 +114,12 @@ func (e *eventhouse) writeSingleTable(metrics []telegraf.Metric) error { // push metrics to a single table format := ingest.FileFormat(ingest.JSON) - err := e.client.PushMetrics(format, e.config.TableName, metricsArray) + err := e.client.PushMetrics(format, e.TableName, metricsArray) return err } func (e *eventhouse) parseconnectionString(cs string) error { // Parse the connection string to extract the endpoint and database - if cs == "" { - return errors.New("connection string must not be empty") - } // Split the connection string into key-value pairs pairs := strings.Split(cs, ";") for _, pair := range pairs { @@ -118,24 +132,27 @@ func (e *eventhouse) parseconnectionString(cs string) error { v = strings.TrimSpace(v) switch k { case "data source", "addr", "address", "network address", "server": - e.config.Endpoint = v + e.Endpoint = v case "initial catalog", "database": - e.config.Database = v + e.Database = v case "ingestion type", "ingestiontype": - e.config.IngestionType = v + e.IngestionType = v case "table name", "tablename": - e.config.TableName = v + e.TableName = v case "create tables", "createtables": - if v == "false" { - e.config.CreateTables = false - } else { - e.config.CreateTables = true + switch v { + case "true": + e.CreateTables = true + case "false": + e.CreateTables = false + default: + return fmt.Errorf("invalid setting %q for %q", v, k) } - case "metrics grouping type, metricsgroupingtype": + case "metrics grouping type", "metricsgroupingtype": if v != adx.TablePerMetric && v != adx.SingleTable { return errors.New("metrics grouping type is not valid:" + v) } - e.config.MetricsGrouping = v + e.MetricsGrouping = v } } return nil diff --git a/plugins/outputs/microsoft_fabric/event_house_test.go b/plugins/outputs/microsoft_fabric/event_house_test.go new file mode 100644 index 0000000000000..9ea15f3b527bd --- /dev/null +++ b/plugins/outputs/microsoft_fabric/event_house_test.go @@ -0,0 +1,234 @@ +package microsoft_fabric + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/influxdata/telegraf/plugins/common/adx" + "github.com/influxdata/telegraf/testutil" +) + +// Helper function to create a test eventhouse instance +func newTestEventHouse(t *testing.T) *eventhouse { + e := &eventhouse{ + log: testutil.Logger{}, + } + err := e.Init() + require.NoError(t, err) + e.Config = adx.Config{} + return e +} + +func TestEventHouse_Init(t *testing.T) { + e := &eventhouse{} + err := e.Init() + require.NoError(t, err) + require.NotNil(t, e.serializer, "serializer should be initialized") + require.True(t, e.CreateTables, "CreateTables should be true by default") +} + +func TestEventHouse_ParseConnectionString(t *testing.T) { + tests := []struct { + name string + connString string + expected *eventhouse + expectedError string + }{ + { + name: "Valid connection string with all parameters", + connString: "data source=https://example.com;database=mydb;table name=mytable;create tables=true;metrics grouping type=tablepermetric", + expected: &eventhouse{ + Config: adx.Config{ + Endpoint: "https://example.com", + Database: "mydb", + TableName: "mytable", + CreateTables: true, + MetricsGrouping: "tablepermetric", + }, + }, + }, + { + name: "Invalid connection string format", + connString: "invalid string format", + expectedError: "invalid connection string format", + }, + { + name: "Case insensitive parameters", + connString: "DATA SOURCE=https://example.com;DATABASE=mydb", + expected: &eventhouse{ + Config: adx.Config{ + Endpoint: "https://example.com", + Database: "mydb", + }, + }, + }, + { + name: "Server parameter instead of data source", + connString: "server=https://example.com;database=mydb", + expected: &eventhouse{ + Config: adx.Config{ + Endpoint: "https://example.com", + Database: "mydb", + }, + }, + }, + { + name: "Invalid metrics grouping type", + connString: "data source=https://example.com;metrics grouping type=Invalid", + expectedError: "metrics grouping type is not valid:Invalid", + }, + { + name: "Create tables parameter true", + connString: "data source=https://example.com;database=mydb;create tables=true", + expected: &eventhouse{ + Config: adx.Config{ + Endpoint: "https://example.com", + Database: "mydb", + CreateTables: true, + }, + }, + }, + { + name: "Create tables parameter false", + connString: "data source=https://example.com;database=mydb;create tables=false", + expected: &eventhouse{ + Config: adx.Config{ + Endpoint: "https://example.com", + Database: "mydb", + CreateTables: false, + }, + }, + }, + { + name: "Invalid create tables value", + connString: "data source=https://example.com;database=mydb;create tables=invalid", + expectedError: "invalid setting", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := newTestEventHouse(t) + err := e.parseconnectionString(tt.connString) + if tt.expectedError != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tt.expectedError) + } else { + require.NoError(t, err) + if tt.expected != nil { + require.Equal(t, tt.expected.Endpoint, e.Endpoint) + require.Equal(t, tt.expected.Database, e.Database) + require.Equal(t, tt.expected.TableName, e.TableName) + require.Equal(t, tt.expected.CreateTables, e.CreateTables) + require.Equal(t, tt.expected.MetricsGrouping, e.MetricsGrouping) + } + } + }) + } +} + +func TestEventHouse_Connect(t *testing.T) { + tests := []struct { + name string + endpoint string + database string + expectError bool + errorContains string + }{ + { + name: "Valid configuration", + endpoint: "https://example.com", + database: "testdb", + }, + { + name: "Empty endpoint", + endpoint: "", + database: "testdb", + expectError: true, + errorContains: "endpoint configuration cannot be empty", + }, + { + name: "Empty database", + endpoint: "https://example.com", + database: "", + expectError: true, + errorContains: "database configuration cannot be empty", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := newTestEventHouse(t) + e.Endpoint = tt.endpoint + e.Database = tt.database + + err := e.Connect() + if tt.expectError { + require.Error(t, err) + if tt.errorContains != "" { + require.Contains(t, err.Error(), tt.errorContains) + } + } else { + require.NoError(t, err) + require.NotNil(t, e.client) + } + }) + } +} + +func TestIsEventhouseEndpoint(t *testing.T) { + tests := []struct { + name string + endpoint string + want bool + }{ + { + name: "Valid data source prefix", + endpoint: "data source=https://example.com", + want: true, + }, + { + name: "Valid address prefix", + endpoint: "address=https://example.com", + want: true, + }, + { + name: "Valid network address prefix", + endpoint: "network address=https://example.com", + want: true, + }, + { + name: "Valid server prefix", + endpoint: "server=https://example.com", + want: true, + }, + { + name: "Invalid prefix", + endpoint: "invalid=https://example.com", + want: false, + }, + { + name: "Empty string", + endpoint: "", + want: false, + }, + { + name: "Just URL", + endpoint: "https://example.com", + want: false, + }, + { + name: "Case insensitive prefix", + endpoint: "DATA SOURCE=https://example.com", + want: true, // isEventhouseEndpoint is not case sensitive + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isEventhouseEndpoint(tt.endpoint) + require.Equal(t, tt.want, got) + }) + } +} diff --git a/plugins/outputs/microsoft_fabric/event_stream.go b/plugins/outputs/microsoft_fabric/event_stream.go index 2281bfdd82717..6cc5edb6379e4 100644 --- a/plugins/outputs/microsoft_fabric/event_stream.go +++ b/plugins/outputs/microsoft_fabric/event_stream.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - "slices" "strconv" "strings" "time" @@ -30,8 +29,6 @@ type eventstream struct { serializer telegraf.Serializer } -var confKeys = []string{"PartitionKey", "MaxMessageSize"} - func (e *eventstream) Init() error { serializer := &json.Serializer{ TimestampUnits: config.Duration(time.Nanosecond), @@ -80,9 +77,8 @@ func (e *eventstream) Write(metrics []telegraf.Metric) error { batchOptions := e.options batches := make(map[string]*azeventhubs.EventDataBatch) - // Cant use `for _, m := range metrics` as we need to move back when a new batch needs to be created - for i := 0; i < len(metrics); i++ { - m := metrics[i] + // Use a range loop with index for readability, while keeping ability to adjust the index + for i, m := range metrics { // Prepare the payload payload, err := e.serializer.Serialize(m) @@ -139,9 +135,8 @@ func (e *eventstream) Write(metrics []telegraf.Metric) error { if err != nil { return fmt.Errorf("creating batch for partition %q failed: %w", partition, err) } - i-- + i -= 1 } - // Send the remaining batches that never exceeded the batch size for partition, batch := range batches { if batch.NumBytes() == 0 { @@ -156,9 +151,6 @@ func (e *eventstream) Write(metrics []telegraf.Metric) error { func (e *eventstream) parseconnectionString(cs string) error { // Parse the connection string - if cs == "" { - return errors.New("connection string must not be empty") - } // Split the connection string into key-value pairs pairs := strings.Split(cs, ";") for _, pair := range pairs { @@ -169,15 +161,18 @@ func (e *eventstream) parseconnectionString(cs string) error { } k = strings.ToLower(strings.TrimSpace(k)) v = strings.TrimSpace(v) - if slices.Contains(confKeys, k) { - switch k { - case "partitionkey", "partition key": - e.partitionKey = v - case "maxmessagesize", "max message size": - if sz, err := strconv.ParseInt(v, 10, 64); err != nil { - e.maxMessageSize = config.Size(sz) - } + + key := strings.ReplaceAll(k, " ", "") + switch key { + case "partitionkey": + e.partitionKey = v + case "maxmessagesize": + sz, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return fmt.Errorf("invalid max message size: %w", err) } + e.maxMessageSize = config.Size(sz) + } } return nil @@ -189,3 +184,7 @@ func (e *eventstream) send(ctx context.Context, batch *azeventhubs.EventDataBatc return e.client.SendEventDataBatch(ctx, batch, nil) } + +func isEventstreamEndpoint(endpoint string) bool { + return strings.HasPrefix(endpoint, "Endpoint=sb") +} diff --git a/plugins/outputs/microsoft_fabric/event_stream_test.go b/plugins/outputs/microsoft_fabric/event_stream_test.go new file mode 100644 index 0000000000000..fb5d2ab29fa30 --- /dev/null +++ b/plugins/outputs/microsoft_fabric/event_stream_test.go @@ -0,0 +1,108 @@ +package microsoft_fabric + +import ( + "strconv" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +// Helper function to create a test eventstream instance +func newTestEventStream(t *testing.T) *eventstream { + e := &eventstream{ + log: testutil.Logger{}, + timeout: config.Duration(time.Second * 5), + options: azeventhubs.EventDataBatchOptions{}, + } + err := e.Init() + require.NoError(t, err) + return e +} + +func TestEventStream_Init(t *testing.T) { + tests := []struct { + name string + maxMessageSize config.Size + expectedMaxSize uint64 + }{ + { + name: "Init with default settings", + maxMessageSize: 0, + expectedMaxSize: 0, + }, + { + name: "Init with custom max message size", + maxMessageSize: 1024, + expectedMaxSize: 1024, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &eventstream{ + maxMessageSize: tt.maxMessageSize, + } + err := e.Init() + require.NoError(t, err) + require.NotNil(t, e.serializer, "serializer should be initialized") + require.Equal(t, tt.expectedMaxSize, e.options.MaxBytes) + }) + } +} + +func TestEventStream_ParseConnectionString(t *testing.T) { + tests := []struct { + name string + connString string + expected map[string]string + expectedError string + }{ + { + name: "Valid connection string with partition key and message size", + connString: "endpoint=https://example.com;partitionkey=mykey;maxmessagesize=1024", + expected: map[string]string{ + "partitionkey": "mykey", + "maxmessagesize": "1024", + }, + }, + { + name: "Invalid connection string format", + connString: "invalid string format", + expectedError: "invalid connection string format", + }, + { + name: "Case insensitive keys", + connString: "PARTITIONKEY=mykey;MaxMessageSize=1024", + expected: map[string]string{ + "partitionkey": "mykey", + "maxmessagesize": "1024", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := newTestEventStream(t) + err := e.parseconnectionString(tt.connString) + + if tt.expectedError != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tt.expectedError) + } else { + require.NoError(t, err) + if tt.expected["partitionkey"] != "" { + require.Equal(t, tt.expected["partitionkey"], e.partitionKey) + } + if tt.expected["maxmessagesize"] != "" { + size, err := strconv.ParseInt(tt.expected["maxmessagesize"], 10, 64) + require.NoError(t, err) + require.Equal(t, config.Size(size), e.maxMessageSize) + } + } + }) + } +} diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric.go b/plugins/outputs/microsoft_fabric/microsoft_fabric.go index 0d3d1a3407046..c257f8b2d4756 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric.go @@ -25,8 +25,8 @@ type fabricOutput interface { type MicrosoftFabric struct { ConnectionString string `toml:"connection_string"` - Log telegraf.Logger `toml:"-"` Timeout config.Duration `toml:"timeout"` + Log telegraf.Logger `toml:"-"` eventhouse *eventhouse eventstream *eventstream @@ -38,19 +38,19 @@ func (*MicrosoftFabric) SampleConfig() string { } func (m *MicrosoftFabric) Init() error { - connectionString := m.ConnectionString - if connectionString == "" { + if m.ConnectionString == "" { return errors.New("endpoint must not be empty") } - if strings.HasPrefix(connectionString, "Endpoint=sb") { + switch { + case isEventstreamEndpoint(m.ConnectionString): m.Log.Info("Detected EventStream endpoint, using EventStream output plugin") eventstream := &eventstream{} - eventstream.connectionString = connectionString + eventstream.connectionString = m.ConnectionString eventstream.log = m.Log eventstream.timeout = m.Timeout - if err := eventstream.parseconnectionString(connectionString); err != nil { + if err := eventstream.parseconnectionString(m.ConnectionString); err != nil { return fmt.Errorf("parsing connection string failed: %w", err) } m.eventstream = eventstream @@ -58,7 +58,7 @@ func (m *MicrosoftFabric) Init() error { return fmt.Errorf("initializing EventStream output failed: %w", err) } m.activePlugin = eventstream - } else if isKustoEndpoint(strings.ToLower(connectionString)) { + case isEventhouseEndpoint(strings.ToLower(m.ConnectionString)): m.Log.Info("Detected EventHouse endpoint, using EventHouse output plugin") // Setting up the AzureDataExplorer plugin initial properties eventhouse := &eventhouse{} @@ -66,57 +66,31 @@ func (m *MicrosoftFabric) Init() error { if err := m.eventhouse.Init(); err != nil { return fmt.Errorf("initializing EventHouse output failed: %w", err) } - eventhouse.config.Endpoint = connectionString + eventhouse.Endpoint = m.ConnectionString eventhouse.log = m.Log - eventhouse.config.Timeout = m.Timeout - if err := eventhouse.parseconnectionString(connectionString); err != nil { + eventhouse.Timeout = m.Timeout + if err := eventhouse.parseconnectionString(m.ConnectionString); err != nil { return fmt.Errorf("parsing connection string failed: %w", err) } m.activePlugin = m.eventhouse - } else { + default: return errors.New("invalid connection string") } return nil } func (m *MicrosoftFabric) Close() error { - if m.activePlugin == nil { - return errors.New("no active plugin to close") - } return m.activePlugin.Close() } func (m *MicrosoftFabric) Connect() error { - if m.activePlugin == nil { - return errors.New("no active plugin to connect") - } return m.activePlugin.Connect() } func (m *MicrosoftFabric) Write(metrics []telegraf.Metric) error { - if m.activePlugin == nil { - return errors.New("no active plugin to write to") - } return m.activePlugin.Write(metrics) } -func isKustoEndpoint(endpoint string) bool { - prefixes := []string{ - "data source=", - "addr=", - "address=", - "network address=", - "server=", - } - - for _, prefix := range prefixes { - if strings.HasPrefix(endpoint, prefix) { - return true - } - } - return false -} - func init() { outputs.Add("microsoft_fabric", func() telegraf.Output { return &MicrosoftFabric{ diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go index 6a036dfb7fa3f..1211af59a885c 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go @@ -4,173 +4,91 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/plugins/common/adx" "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" ) -type MockOutput struct { - mock.Mock -} - -func (m *MockOutput) Connect() error { - args := m.Called() - return args.Error(0) -} - -func (m *MockOutput) Close() error { - args := m.Called() - return args.Error(0) -} - -func (m *MockOutput) Write(metrics []telegraf.Metric) error { - args := m.Called(metrics) - return args.Error(0) -} - -func (m *MockOutput) Init() error { - args := m.Called() - return args.Error(0) -} - -func TestMicrosoftFabric_Connect(t *testing.T) { - mockOutput := new(MockOutput) - mockOutput.On("Connect").Return(nil) - - plugin := MicrosoftFabric{ - activePlugin: mockOutput, - } - - err := plugin.Connect() - require.NoError(t, err) - mockOutput.AssertExpectations(t) -} - -func TestMicrosoftFabric_Connect_Err(t *testing.T) { - plugin := MicrosoftFabric{} - err := plugin.Connect() - require.Equal(t, "no active plugin to connect", err.Error()) -} - -func TestMicrosoftFabric_Close(t *testing.T) { - mockOutput := new(MockOutput) - mockOutput.On("Close").Return(nil) - - plugin := MicrosoftFabric{ - activePlugin: mockOutput, - } - - err := plugin.Close() - require.NoError(t, err) - mockOutput.AssertExpectations(t) -} - -func TestMicrosoftFabric_Close_Err(t *testing.T) { - plugin := MicrosoftFabric{} - err := plugin.Close() - require.Equal(t, "no active plugin to close", err.Error()) -} - -func TestMicrosoftFabric_Write(t *testing.T) { - mockOutput := new(MockOutput) - mockOutput.On("Write", mock.Anything).Return(nil) - - plugin := MicrosoftFabric{ - activePlugin: mockOutput, - } - - metrics := []telegraf.Metric{ - testutil.TestMetric(1.0, "test_metric"), - } - - err := plugin.Write(metrics) - require.NoError(t, err) - mockOutput.AssertExpectations(t) -} - -func TestMicrosoftFabric_Write_Err(t *testing.T) { - plugin := MicrosoftFabric{} - - metrics := []telegraf.Metric{ - testutil.TestMetric(1.0, "test_metric"), - } - - err := plugin.Write(metrics) - require.Equal(t, "no active plugin to write to", err.Error()) -} - -func TestIsKustoEndpoint(t *testing.T) { - testCases := []struct { - name string - endpoint string - expected bool +func TestInit(t *testing.T) { + tests := []struct { + name string + connectionString string + timeout config.Duration + expectPlugin string // "eventstream" or "eventhouse" + expectError bool + errorContains string + initFunc func(*MicrosoftFabric) // For custom initialization if needed }{ { - name: "Valid address prefix", - endpoint: "address=https://example.com", - expected: true, + name: "Empty connection string", + connectionString: "", + expectError: true, + errorContains: "endpoint must not be empty", }, { - name: "Valid network address prefix", - endpoint: "network address=https://example.com", - expected: true, + name: "Valid EventStream connection", + connectionString: "Endpoint=sb://namespace.servicebus.windows.net/;SharedAccessKeyName=keyName;SharedAccessKey=key", + timeout: config.Duration(30 * time.Second), + expectPlugin: "eventstream", }, { - name: "Valid server prefix", - endpoint: "server=https://example.com", - expected: true, + name: "Valid EventHouse connection", + connectionString: "data source=https://example.kusto.windows.net;Database=db", + timeout: config.Duration(30 * time.Second), + expectPlugin: "eventhouse", }, { - name: "Invalid prefix", - endpoint: "https://example.com", - expected: false, + name: "Invalid connection string format", + connectionString: "invalid=format", + expectError: true, + errorContains: "invalid connection string", }, { - name: "Empty endpoint", - endpoint: "", - expected: false, + name: "EventStream connection string parsing error", + connectionString: "Endpoint=sb://namespace.servicebus.windows.net/;invalid_param", + expectError: true, + errorContains: "parsing connection string failed", + initFunc: func(mf *MicrosoftFabric) { + mf.eventstream = &eventstream{} + }, }, - } - - for _, tC := range testCases { - t.Run(tC.name, func(t *testing.T) { - result := isKustoEndpoint(tC.endpoint) - require.Equal(t, tC.expected, result) - }) - } -} - -func TestMicrosoftFabric_Init(t *testing.T) { - tests := []struct { - name string - connectionString string - expectedError string - }{ { - name: "Empty connection string", - connectionString: "", - expectedError: "endpoint must not be empty", + name: "EventHouse connection string parsing error", + connectionString: "data source=https://example.kusto.windows.net;invalid_param", + expectError: true, + errorContains: "parsing connection string failed", + initFunc: func(mf *MicrosoftFabric) { + mf.eventhouse = &eventhouse{} + }, }, { - name: "Invalid connection string", - connectionString: "invalid_connection_string", - expectedError: "invalid connection string", + name: "Malformed connection string", + connectionString: "endpoint=;key=;", + expectError: true, + errorContains: "invalid connection string", }, { - name: "Valid EventHouse connection string", - connectionString: "Endpoint=sb://namespace.servicebus.windows.net/;" + - "SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=superSecret1234;EntityPath=hubName", - expectedError: "", + name: "EventStream with custom timeout", + connectionString: "Endpoint=sb://namespace.servicebus.windows.net/;SharedAccessKeyName=keyName;SharedAccessKey=key", + timeout: config.Duration(60 * time.Second), + expectPlugin: "eventstream", + initFunc: func(mf *MicrosoftFabric) { + mf.Timeout = config.Duration(60 * time.Second) + }, }, { - name: "Valid Kusto connection string", - connectionString: "data source=https://example.kusto.windows.net;Database=e2e", - expectedError: "", + name: "EventHouse with database configuration", + connectionString: "data source=https://example.kusto.windows.net;Database=testdb", + timeout: config.Duration(30 * time.Second), + expectPlugin: "eventhouse", + initFunc: func(mf *MicrosoftFabric) { + mf.eventhouse = &eventhouse{ + Config: adx.Config{ + Database: "testdb", + }, + } + }, }, } @@ -179,21 +97,45 @@ func TestMicrosoftFabric_Init(t *testing.T) { mf := &MicrosoftFabric{ ConnectionString: tt.connectionString, Log: testutil.Logger{}, - eventhouse: &eventhouse{ - config: &adx.Config{ - Database: "database", - }, - }, - eventstream: &eventstream{ - timeout: config.Duration(30 * time.Second), - }, + Timeout: tt.timeout, } + + // Apply custom initialization if provided + if tt.initFunc != nil { + tt.initFunc(mf) + } + err := mf.Init() - if tt.expectedError != "" { + + if tt.expectError { require.Error(t, err) - assert.Equal(t, tt.expectedError, err.Error()) - } else { - require.NoError(t, err) + if tt.errorContains != "" { + require.Contains(t, err.Error(), tt.errorContains) + } + return + } + + require.NoError(t, err) + require.NotNil(t, mf.activePlugin, "Active plugin should be set") + + // Verify correct plugin type was selected + switch tt.expectPlugin { + case "eventstream": + require.NotNil(t, mf.eventstream, "EventStream should be initialized") + require.Equal(t, mf.eventstream, mf.activePlugin) + case "eventhouse": + require.NotNil(t, mf.eventhouse, "EventHouse should be initialized") + require.Equal(t, mf.eventhouse, mf.activePlugin) + } + + // Verify timeout was properly set + if tt.timeout > 0 { + switch p := mf.activePlugin.(type) { + case *eventstream: + require.Equal(t, tt.timeout, p.timeout) + case *eventhouse: + require.Equal(t, tt.timeout, p.Timeout) + } } }) } diff --git a/plugins/outputs/microsoft_fabric/sample.conf b/plugins/outputs/microsoft_fabric/sample.conf index 23a8ac906b746..57bc58ed19089 100644 --- a/plugins/outputs/microsoft_fabric/sample.conf +++ b/plugins/outputs/microsoft_fabric/sample.conf @@ -1,64 +1,7 @@ # Sends metrics to Microsoft Fabric [[outputs.microsoft_fabric]] - ## The URI property of the Eventhouse resource on Azure - ## ex: connection_string = "Data Source=https://myadxresource.australiasoutheast.kusto.windows.net" - connection_string = "" + ## The URI property of the resource on Azure + connection_string = "https://trd-abcd.xx.kusto.fabric.microsoft.com;Database=kusto_eh; Table Name=telegraf_dump;Key=value" - - ## Using this section the plugin will send metrics to an Eventhouse endpoint - ## for ingesting, storing, and querying large volumes of data with low latency. - [outputs.microsoft_fabric.eventhouse] - [outputs.microsoft_fabric.eventhouse.cluster_config] - ## Database metrics will be written to - ## NOTE: The plugin will NOT generate the database. It is expected the database already exists. - database = "" - - ## Timeout for Eventhouse operations - # timeout = "20s" - - ## Type of metrics grouping; available options are: - ## tablepermetric -- for one table per distinct metric - ## singletable -- for writing all metrics to the same table - # metrics_grouping_type = "tablepermetric" - - # Name of the table to store metrics - ## NOTE: This option is only used for "singletable" metrics grouping - # table_name = "" - - ## Creates tables and relevant mapping - ## Disable when running with the lowest possible permissions i.e. table ingestor role. - # create_tables = true - - ## Ingestion method to use; available options are - ## - managed -- streaming ingestion with fallback to batched ingestion or the "queued" method below - ## - queued -- queue up metrics data and process sequentially - # ingestion_type = "queued" - - ## Using this section the plugin will send metrics to an EventStream endpoint - ## for transforming and routing metrics to various destinations without writing - ## any code. - [outputs.microsoft_fabric.eventstream] - ## The full connection string to the Event Hub (required) - ## The shared access key must have "Send" permissions on the target Event Hub. - - ## Client timeout - # timeout = "30s" - - ## Partition key - ## Metric tag or field name to use for the event partition key. The value of - ## this tag or field is set as the key for events if it exists. If both, tag - ## and field, exist the tag is preferred. - # partition_key = "" - - ## Set the maximum batch message size in bytes - ## The allowable size depends on the Event Hub tier; not setting this option or setting - ## it to zero will use the default size of the Azure Event Hubs Client library. See - ## https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas#basic-vs-standard-vs-premium-vs-dedicated-tiers - ## for the allowable size of your tier. - # max_message_size = "0B" - - ## Data format to output. - ## Each data format has its own unique set of configuration options, read - ## more about them here: - ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md - data_format = "json" \ No newline at end of file + ## Client timeout + #timeout = "30s" \ No newline at end of file From 48e2652a416fe191b62964fd02f3c0939f8d5859 Mon Sep 17 00:00:00 2001 From: asaharn Date: Tue, 3 Jun 2025 00:43:29 +0530 Subject: [PATCH 19/28] Lint changes --- plugins/outputs/microsoft_fabric/event_stream_test.go | 3 ++- plugins/outputs/microsoft_fabric/microsoft_fabric_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/event_stream_test.go b/plugins/outputs/microsoft_fabric/event_stream_test.go index fb5d2ab29fa30..2d6fe117deec7 100644 --- a/plugins/outputs/microsoft_fabric/event_stream_test.go +++ b/plugins/outputs/microsoft_fabric/event_stream_test.go @@ -6,9 +6,10 @@ import ( "time" "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" + "github.com/stretchr/testify/require" + "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/testutil" - "github.com/stretchr/testify/require" ) // Helper function to create a test eventstream instance diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go index 1211af59a885c..d0c00e0a8d2a8 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go @@ -4,10 +4,11 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/plugins/common/adx" "github.com/influxdata/telegraf/testutil" - "github.com/stretchr/testify/require" ) func TestInit(t *testing.T) { From 9794317024c75d23ce93953ed89154c0e514b983 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Tue, 3 Jun 2025 14:19:26 +0200 Subject: [PATCH 20/28] Include sample.conf in README --- plugins/outputs/microsoft_fabric/README.md | 6 +++--- plugins/outputs/microsoft_fabric/sample.conf | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/README.md b/plugins/outputs/microsoft_fabric/README.md index e940cd70eccde..3b02edf2179fd 100644 --- a/plugins/outputs/microsoft_fabric/README.md +++ b/plugins/outputs/microsoft_fabric/README.md @@ -19,14 +19,14 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. ## Configuration -```toml +```toml @sample.conf # Sends metrics to Microsoft Fabric [[outputs.microsoft_fabric]] ## The URI property of the resource on Azure connection_string = "https://trd-abcd.xx.kusto.fabric.microsoft.com;Database=kusto_eh; Table Name=telegraf_dump;Key=value" ## Client timeout - timeout = "30s" + # timeout = "30s" ``` ### Connection String @@ -133,7 +133,7 @@ about the merge process check the [`.create-merge` documentation][create-merge]. Eventstreams allow you to bring real-time events into Fabric, transform them, and then route them to various destinations without writing any code (no-code). -For more information, visit the [Eventstream documentation][eventstream_docs]. +For more information, visit the [Eventstream documentation][eventstream_docs]. To communicate with an eventstream, you need a connection string for the namespace or the event hub. If you use a connection string to the namespace diff --git a/plugins/outputs/microsoft_fabric/sample.conf b/plugins/outputs/microsoft_fabric/sample.conf index 57bc58ed19089..4f935189c99ac 100644 --- a/plugins/outputs/microsoft_fabric/sample.conf +++ b/plugins/outputs/microsoft_fabric/sample.conf @@ -4,4 +4,4 @@ connection_string = "https://trd-abcd.xx.kusto.fabric.microsoft.com;Database=kusto_eh; Table Name=telegraf_dump;Key=value" ## Client timeout - #timeout = "30s" \ No newline at end of file + # timeout = "30s" From 65880650b33f109320b2a915e45e8d0f9ac9c87d Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Tue, 3 Jun 2025 14:52:33 +0200 Subject: [PATCH 21/28] Cleanup plugin tests --- .../outputs/microsoft_fabric/event_stream.go | 13 +- .../microsoft_fabric/microsoft_fabric_test.go | 206 +++++++++--------- 2 files changed, 116 insertions(+), 103 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/event_stream.go b/plugins/outputs/microsoft_fabric/event_stream.go index 6cc5edb6379e4..5000a48019511 100644 --- a/plugins/outputs/microsoft_fabric/event_stream.go +++ b/plugins/outputs/microsoft_fabric/event_stream.go @@ -18,15 +18,16 @@ import ( ) type eventstream struct { + connectionString string + timeout config.Duration + log telegraf.Logger + partitionKey string maxMessageSize config.Size - timeout config.Duration - connectionString string - log telegraf.Logger - client *azeventhubs.ProducerClient - options azeventhubs.EventDataBatchOptions - serializer telegraf.Serializer + client *azeventhubs.ProducerClient + options azeventhubs.EventDataBatchOptions + serializer telegraf.Serializer } func (e *eventstream) Init() error { diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go index d0c00e0a8d2a8..52fd481409df6 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go @@ -11,133 +11,145 @@ import ( "github.com/influxdata/telegraf/testutil" ) -func TestInit(t *testing.T) { +func TestInitFail(t *testing.T) { tests := []struct { - name string - connectionString string - timeout config.Duration - expectPlugin string // "eventstream" or "eventhouse" - expectError bool - errorContains string - initFunc func(*MicrosoftFabric) // For custom initialization if needed + name string + connection string + expected string }{ { - name: "Empty connection string", - connectionString: "", - expectError: true, - errorContains: "endpoint must not be empty", + name: "empty connection string", + expected: "endpoint must not be empty", }, { - name: "Valid EventStream connection", - connectionString: "Endpoint=sb://namespace.servicebus.windows.net/;SharedAccessKeyName=keyName;SharedAccessKey=key", - timeout: config.Duration(30 * time.Second), - expectPlugin: "eventstream", + name: "invalid connection string format", + connection: "invalid=format", + expected: "invalid connection string", }, { - name: "Valid EventHouse connection", - connectionString: "data source=https://example.kusto.windows.net;Database=db", - timeout: config.Duration(30 * time.Second), - expectPlugin: "eventhouse", + name: "Malformed connection string", + connection: "endpoint=;key=;", + expected: "invalid connection string", }, { - name: "Invalid connection string format", - connectionString: "invalid=format", - expectError: true, - errorContains: "invalid connection string", + name: "invalid eventhouse connection string", + connection: "data source=https://example.kusto.windows.net;invalid_param", + expected: "parsing connection string failed", }, { - name: "EventStream connection string parsing error", - connectionString: "Endpoint=sb://namespace.servicebus.windows.net/;invalid_param", - expectError: true, - errorContains: "parsing connection string failed", - initFunc: func(mf *MicrosoftFabric) { - mf.eventstream = &eventstream{} - }, + name: "invalid eventstream connection string", + connection: "Endpoint=sb://namespace.servicebus.windows.net/;invalid_param", + expected: "parsing connection string failed", }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup the plugin + plugin := &MicrosoftFabric{ + ConnectionString: tt.connection, + Log: testutil.Logger{}, + } + + // Check the returned error + require.ErrorContains(t, plugin.Init(), tt.expected) + }) + } +} + +func TestInitEventHouse(t *testing.T) { + tests := []struct { + name string + connection string + timeout config.Duration + expected adx.Config + }{ { - name: "EventHouse connection string parsing error", - connectionString: "data source=https://example.kusto.windows.net;invalid_param", - expectError: true, - errorContains: "parsing connection string failed", - initFunc: func(mf *MicrosoftFabric) { - mf.eventhouse = &eventhouse{} + name: "valid configuration", + connection: "data source=https://example.kusto.windows.net;Database=testdb", + expected: adx.Config{ + Endpoint: "https://example.kusto.windows.net", + Database: "testdb", + CreateTables: true, + Timeout: config.Duration(30 * time.Second), }, }, { - name: "Malformed connection string", - connectionString: "endpoint=;key=;", - expectError: true, - errorContains: "invalid connection string", + name: "valid configuration with timeout", + connection: "data source=https://example.kusto.windows.net;Database=testdb", + timeout: config.Duration(60 * time.Second), + expected: adx.Config{ + Endpoint: "https://example.kusto.windows.net", + Database: "testdb", + CreateTables: true, + Timeout: config.Duration(60 * time.Second), + }, }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup the plugin + plugin := &MicrosoftFabric{ + ConnectionString: tt.connection, + Timeout: tt.timeout, + Log: testutil.Logger{}, + } + require.NoError(t, plugin.Init()) + + // Check the created plugin + require.NotNil(t, plugin.activePlugin, "active plugin should have been set") + ap, ok := plugin.activePlugin.(*eventhouse) + require.Truef(t, ok, "expected evenhouse plugin but got %T", plugin.activePlugin) + require.Equal(t, tt.expected, ap.Config) + }) + } +} + +func TestInitEventStream(t *testing.T) { + tests := []struct { + name string + connection string + timeout config.Duration + expected eventstream + }{ { - name: "EventStream with custom timeout", - connectionString: "Endpoint=sb://namespace.servicebus.windows.net/;SharedAccessKeyName=keyName;SharedAccessKey=key", - timeout: config.Duration(60 * time.Second), - expectPlugin: "eventstream", - initFunc: func(mf *MicrosoftFabric) { - mf.Timeout = config.Duration(60 * time.Second) + name: "valid connection", + connection: "Endpoint=sb://namespace.servicebus.windows.net/;SharedAccessKeyName=keyName;SharedAccessKey=key", + expected: eventstream{ + connectionString: "Endpoint=sb://namespace.servicebus.windows.net/;SharedAccessKeyName=keyName;SharedAccessKey=key", + timeout: config.Duration(30 * time.Second), }, }, { - name: "EventHouse with database configuration", - connectionString: "data source=https://example.kusto.windows.net;Database=testdb", - timeout: config.Duration(30 * time.Second), - expectPlugin: "eventhouse", - initFunc: func(mf *MicrosoftFabric) { - mf.eventhouse = &eventhouse{ - Config: adx.Config{ - Database: "testdb", - }, - } + name: "valid connection with timeout", + connection: "Endpoint=sb://namespace.servicebus.windows.net/;SharedAccessKeyName=keyName;SharedAccessKey=key", + timeout: config.Duration(60 * time.Second), + expected: eventstream{ + connectionString: "Endpoint=sb://namespace.servicebus.windows.net/;SharedAccessKeyName=keyName;SharedAccessKey=key", + timeout: config.Duration(30 * time.Second), }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - mf := &MicrosoftFabric{ - ConnectionString: tt.connectionString, - Log: testutil.Logger{}, + // Setup plugin + plugin := &MicrosoftFabric{ + ConnectionString: tt.connection, Timeout: tt.timeout, + Log: testutil.Logger{}, } + require.NoError(t, plugin.Init()) - // Apply custom initialization if provided - if tt.initFunc != nil { - tt.initFunc(mf) - } - - err := mf.Init() - - if tt.expectError { - require.Error(t, err) - if tt.errorContains != "" { - require.Contains(t, err.Error(), tt.errorContains) - } - return - } - - require.NoError(t, err) - require.NotNil(t, mf.activePlugin, "Active plugin should be set") - - // Verify correct plugin type was selected - switch tt.expectPlugin { - case "eventstream": - require.NotNil(t, mf.eventstream, "EventStream should be initialized") - require.Equal(t, mf.eventstream, mf.activePlugin) - case "eventhouse": - require.NotNil(t, mf.eventhouse, "EventHouse should be initialized") - require.Equal(t, mf.eventhouse, mf.activePlugin) - } - - // Verify timeout was properly set - if tt.timeout > 0 { - switch p := mf.activePlugin.(type) { - case *eventstream: - require.Equal(t, tt.timeout, p.timeout) - case *eventhouse: - require.Equal(t, tt.timeout, p.Timeout) - } - } + // Check the created plugin + require.NotNil(t, plugin.activePlugin, "active plugin should have been set") + ap, ok := plugin.activePlugin.(*eventstream) + require.Truef(t, ok, "expected evenstream plugin but got %T", plugin.activePlugin) + require.Equal(t, tt.expected.connectionString, ap.connectionString) + require.Equal(t, tt.expected.timeout, ap.timeout) + require.Equal(t, tt.expected.partitionKey, ap.partitionKey) + require.Equal(t, tt.expected.maxMessageSize, ap.maxMessageSize) }) } } From 82649091a96b1d4b22f2708beec3c521fd37d6d5 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Tue, 3 Jun 2025 15:10:59 +0200 Subject: [PATCH 22/28] Cleanup plugin tests --- .../outputs/microsoft_fabric/event_house.go | 1 + .../microsoft_fabric/event_house_test.go | 224 ++++++------------ .../microsoft_fabric/microsoft_fabric_test.go | 60 +++++ 3 files changed, 127 insertions(+), 158 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/event_house.go b/plugins/outputs/microsoft_fabric/event_house.go index c6702e9e1e060..a13ee9203f5f4 100644 --- a/plugins/outputs/microsoft_fabric/event_house.go +++ b/plugins/outputs/microsoft_fabric/event_house.go @@ -54,6 +54,7 @@ func isEventhouseEndpoint(endpoint string) bool { } func (e *eventhouse) Connect() error { + fmt.Printf("cfg: %+v\n", e.Config) client, err := e.NewClient("Kusto.Telegraf", e.log) if err != nil { return fmt.Errorf("creating new client failed: %w", err) diff --git a/plugins/outputs/microsoft_fabric/event_house_test.go b/plugins/outputs/microsoft_fabric/event_house_test.go index 9ea15f3b527bd..a0a520d1039a9 100644 --- a/plugins/outputs/microsoft_fabric/event_house_test.go +++ b/plugins/outputs/microsoft_fabric/event_house_test.go @@ -9,170 +9,74 @@ import ( "github.com/influxdata/telegraf/testutil" ) -// Helper function to create a test eventhouse instance -func newTestEventHouse(t *testing.T) *eventhouse { - e := &eventhouse{ - log: testutil.Logger{}, - } - err := e.Init() - require.NoError(t, err) - e.Config = adx.Config{} - return e -} - -func TestEventHouse_Init(t *testing.T) { - e := &eventhouse{} - err := e.Init() - require.NoError(t, err) - require.NotNil(t, e.serializer, "serializer should be initialized") - require.True(t, e.CreateTables, "CreateTables should be true by default") -} - -func TestEventHouse_ParseConnectionString(t *testing.T) { +func TestEventHouseConnectSuccess(t *testing.T) { tests := []struct { - name string - connString string - expected *eventhouse - expectedError string + name string + endpoint string + database string }{ { - name: "Valid connection string with all parameters", - connString: "data source=https://example.com;database=mydb;table name=mytable;create tables=true;metrics grouping type=tablepermetric", - expected: &eventhouse{ - Config: adx.Config{ - Endpoint: "https://example.com", - Database: "mydb", - TableName: "mytable", - CreateTables: true, - MetricsGrouping: "tablepermetric", - }, - }, - }, - { - name: "Invalid connection string format", - connString: "invalid string format", - expectedError: "invalid connection string format", - }, - { - name: "Case insensitive parameters", - connString: "DATA SOURCE=https://example.com;DATABASE=mydb", - expected: &eventhouse{ - Config: adx.Config{ - Endpoint: "https://example.com", - Database: "mydb", - }, - }, - }, - { - name: "Server parameter instead of data source", - connString: "server=https://example.com;database=mydb", - expected: &eventhouse{ - Config: adx.Config{ - Endpoint: "https://example.com", - Database: "mydb", - }, - }, - }, - { - name: "Invalid metrics grouping type", - connString: "data source=https://example.com;metrics grouping type=Invalid", - expectedError: "metrics grouping type is not valid:Invalid", - }, - { - name: "Create tables parameter true", - connString: "data source=https://example.com;database=mydb;create tables=true", - expected: &eventhouse{ - Config: adx.Config{ - Endpoint: "https://example.com", - Database: "mydb", - CreateTables: true, - }, - }, - }, - { - name: "Create tables parameter false", - connString: "data source=https://example.com;database=mydb;create tables=false", - expected: &eventhouse{ - Config: adx.Config{ - Endpoint: "https://example.com", - Database: "mydb", - CreateTables: false, - }, - }, - }, - { - name: "Invalid create tables value", - connString: "data source=https://example.com;database=mydb;create tables=invalid", - expectedError: "invalid setting", + name: "valid configuration", + endpoint: "https://example.com", + database: "testdb", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - e := newTestEventHouse(t) - err := e.parseconnectionString(tt.connString) - if tt.expectedError != "" { - require.Error(t, err) - require.Contains(t, err.Error(), tt.expectedError) - } else { - require.NoError(t, err) - if tt.expected != nil { - require.Equal(t, tt.expected.Endpoint, e.Endpoint) - require.Equal(t, tt.expected.Database, e.Database) - require.Equal(t, tt.expected.TableName, e.TableName) - require.Equal(t, tt.expected.CreateTables, e.CreateTables) - require.Equal(t, tt.expected.MetricsGrouping, e.MetricsGrouping) - } + // Setup plugin + plugin := &eventhouse{ + Config: adx.Config{ + Endpoint: tt.endpoint, + Database: tt.database, + }, + log: testutil.Logger{}, } + require.NoError(t, plugin.Init()) + + // Check for successful connection and client creation + require.NoError(t, plugin.Connect()) + require.NotNil(t, plugin.client) }) } } -func TestEventHouse_Connect(t *testing.T) { +func TestEventHouseConnectFail(t *testing.T) { tests := []struct { - name string - endpoint string - database string - expectError bool - errorContains string + name string + endpoint string + database string + expected string }{ { - name: "Valid configuration", - endpoint: "https://example.com", + name: "empty endpoint", + endpoint: "", database: "testdb", + expected: "endpoint configuration cannot be empty", }, { - name: "Empty endpoint", - endpoint: "", - database: "testdb", - expectError: true, - errorContains: "endpoint configuration cannot be empty", - }, - { - name: "Empty database", - endpoint: "https://example.com", - database: "", - expectError: true, - errorContains: "database configuration cannot be empty", + name: "empty database", + endpoint: "https://example.com", + database: "", + expected: "database configuration cannot be empty", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - e := newTestEventHouse(t) - e.Endpoint = tt.endpoint - e.Database = tt.database - - err := e.Connect() - if tt.expectError { - require.Error(t, err) - if tt.errorContains != "" { - require.Contains(t, err.Error(), tt.errorContains) - } - } else { - require.NoError(t, err) - require.NotNil(t, e.client) + // Setup plugin + plugin := &eventhouse{ + Config: adx.Config{ + Endpoint: tt.endpoint, + Database: tt.database, + }, + log: testutil.Logger{}, } + require.NoError(t, plugin.Init()) + + // Connect should fail + require.ErrorContains(t, plugin.Connect(), tt.expected) + require.Nil(t, plugin.client) }) } } @@ -181,54 +85,58 @@ func TestIsEventhouseEndpoint(t *testing.T) { tests := []struct { name string endpoint string - want bool }{ { - name: "Valid data source prefix", + name: "data source prefix", endpoint: "data source=https://example.com", - want: true, }, { - name: "Valid address prefix", + name: "address prefix", endpoint: "address=https://example.com", - want: true, }, { - name: "Valid network address prefix", + name: "network address prefix", endpoint: "network address=https://example.com", - want: true, }, { - name: "Valid server prefix", + name: "server prefix", endpoint: "server=https://example.com", - want: true, }, + { + name: "case insensitive prefix", + endpoint: "DATA SOURCE=https://example.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.True(t, isEventhouseEndpoint(tt.endpoint)) + }) + } +} + +func TestIsNotEventhouseEndpoint(t *testing.T) { + tests := []struct { + name string + endpoint string + }{ { name: "Invalid prefix", endpoint: "invalid=https://example.com", - want: false, }, { name: "Empty string", endpoint: "", - want: false, }, { name: "Just URL", endpoint: "https://example.com", - want: false, - }, - { - name: "Case insensitive prefix", - endpoint: "DATA SOURCE=https://example.com", - want: true, // isEventhouseEndpoint is not case sensitive }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := isEventhouseEndpoint(tt.endpoint) - require.Equal(t, tt.want, got) + require.False(t, isEventhouseEndpoint(tt.endpoint)) }) } } diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go index 52fd481409df6..7c4817f94e8b6 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go @@ -36,6 +36,21 @@ func TestInitFail(t *testing.T) { connection: "data source=https://example.kusto.windows.net;invalid_param", expected: "parsing connection string failed", }, + { + name: "invalid eventhouse connection string format", + connection: "invalid string format", + expected: "invalid connection string format", + }, + { + name: "invalid eventhouse metrics grouping type", + connection: "data source=https://example.com;metrics grouping type=Invalid", + expected: "metrics grouping type is not valid:Invalid", + }, + { + name: "invalid eventhouse create tables value", + connection: "data source=https://example.com;database=mydb;create tables=invalid", + expected: "invalid setting", + }, { name: "invalid eventstream connection string", connection: "Endpoint=sb://namespace.servicebus.windows.net/;invalid_param", @@ -85,6 +100,51 @@ func TestInitEventHouse(t *testing.T) { Timeout: config.Duration(60 * time.Second), }, }, + { + name: "valid connection string with all parameters", + connection: "data source=https://example.com;database=mydb;table name=mytable;create tables=true;metrics grouping type=tablepermetric", + expected: adx.Config{ + Endpoint: "https://example.com", + Database: "mydb", + TableName: "mytable", + CreateTables: true, + MetricsGrouping: "tablepermetric", + }, + }, + { + name: "case insensitive parameters", + connection: "DATA SOURCE=https://example.com;DATABASE=mydb", + expected: adx.Config{ + Endpoint: "https://example.com", + Database: "mydb", + }, + }, + { + name: "server parameter instead of data source", + connection: "server=https://example.com;database=mydb", + expected: adx.Config{ + Endpoint: "https://example.com", + Database: "mydb", + }, + }, + { + name: "create tables parameter true", + connection: "data source=https://example.com;database=mydb;create tables=true", + expected: adx.Config{ + Endpoint: "https://example.com", + Database: "mydb", + CreateTables: true, + }, + }, + { + name: "create tables parameter false", + connection: "data source=https://example.com;database=mydb;create tables=false", + expected: adx.Config{ + Endpoint: "https://example.com", + Database: "mydb", + CreateTables: false, + }, + }, } for _, tt := range tests { From 2750c5e9c8d2f32426b627a5fbd37b052d062990 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Tue, 3 Jun 2025 15:19:05 +0200 Subject: [PATCH 23/28] Cleanup plugin tests --- .../microsoft_fabric/event_house_test.go | 4 + .../microsoft_fabric/event_stream_test.go | 94 +++++-------------- .../microsoft_fabric/microsoft_fabric_test.go | 33 ++++++- 3 files changed, 55 insertions(+), 76 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/event_house_test.go b/plugins/outputs/microsoft_fabric/event_house_test.go index a0a520d1039a9..d8545541ffac0 100644 --- a/plugins/outputs/microsoft_fabric/event_house_test.go +++ b/plugins/outputs/microsoft_fabric/event_house_test.go @@ -132,6 +132,10 @@ func TestIsNotEventhouseEndpoint(t *testing.T) { name: "Just URL", endpoint: "https://example.com", }, + { + name: "eventstream endpoint", + endpoint: "Endpoint=sb://example.com", + }, } for _, tt := range tests { diff --git a/plugins/outputs/microsoft_fabric/event_stream_test.go b/plugins/outputs/microsoft_fabric/event_stream_test.go index 2d6fe117deec7..4bfd1e6eb9180 100644 --- a/plugins/outputs/microsoft_fabric/event_stream_test.go +++ b/plugins/outputs/microsoft_fabric/event_stream_test.go @@ -1,109 +1,59 @@ package microsoft_fabric import ( - "strconv" "testing" - "time" - "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" "github.com/stretchr/testify/require" - - "github.com/influxdata/telegraf/config" - "github.com/influxdata/telegraf/testutil" ) -// Helper function to create a test eventstream instance -func newTestEventStream(t *testing.T) *eventstream { - e := &eventstream{ - log: testutil.Logger{}, - timeout: config.Duration(time.Second * 5), - options: azeventhubs.EventDataBatchOptions{}, - } - err := e.Init() - require.NoError(t, err) - return e -} - -func TestEventStream_Init(t *testing.T) { +func TestIsEventstreamEndpoint(t *testing.T) { tests := []struct { - name string - maxMessageSize config.Size - expectedMaxSize uint64 + name string + endpoint string }{ { - name: "Init with default settings", - maxMessageSize: 0, - expectedMaxSize: 0, + name: "endpoint prefix", + endpoint: "Endpoint=sb://example.com", }, { - name: "Init with custom max message size", - maxMessageSize: 1024, - expectedMaxSize: 1024, + name: "case insensitive prefix", + endpoint: "Endpoint=sb://example.com", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - e := &eventstream{ - maxMessageSize: tt.maxMessageSize, - } - err := e.Init() - require.NoError(t, err) - require.NotNil(t, e.serializer, "serializer should be initialized") - require.Equal(t, tt.expectedMaxSize, e.options.MaxBytes) + require.True(t, isEventstreamEndpoint(tt.endpoint)) }) } } -func TestEventStream_ParseConnectionString(t *testing.T) { +func TestIsNotEventstreamEndpoint(t *testing.T) { tests := []struct { - name string - connString string - expected map[string]string - expectedError string + name string + endpoint string }{ { - name: "Valid connection string with partition key and message size", - connString: "endpoint=https://example.com;partitionkey=mykey;maxmessagesize=1024", - expected: map[string]string{ - "partitionkey": "mykey", - "maxmessagesize": "1024", - }, + name: "Invalid prefix", + endpoint: "invalid=https://example.com", }, { - name: "Invalid connection string format", - connString: "invalid string format", - expectedError: "invalid connection string format", + name: "Empty string", + endpoint: "", }, { - name: "Case insensitive keys", - connString: "PARTITIONKEY=mykey;MaxMessageSize=1024", - expected: map[string]string{ - "partitionkey": "mykey", - "maxmessagesize": "1024", - }, + name: "Just URL", + endpoint: "https://example.com", + }, + { + name: "eventhouse endpoint", + endpoint: "data source=https://example.com", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - e := newTestEventStream(t) - err := e.parseconnectionString(tt.connString) - - if tt.expectedError != "" { - require.Error(t, err) - require.Contains(t, err.Error(), tt.expectedError) - } else { - require.NoError(t, err) - if tt.expected["partitionkey"] != "" { - require.Equal(t, tt.expected["partitionkey"], e.partitionKey) - } - if tt.expected["maxmessagesize"] != "" { - size, err := strconv.ParseInt(tt.expected["maxmessagesize"], 10, 64) - require.NoError(t, err) - require.Equal(t, config.Size(size), e.maxMessageSize) - } - } + require.False(t, isEventstreamEndpoint(tt.endpoint)) }) } } diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go index 7c4817f94e8b6..f4153dd809de1 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs" "github.com/stretchr/testify/require" "github.com/influxdata/telegraf/config" @@ -56,6 +57,11 @@ func TestInitFail(t *testing.T) { connection: "Endpoint=sb://namespace.servicebus.windows.net/;invalid_param", expected: "parsing connection string failed", }, + { + name: "invalid eventstream connection string format", + connection: "invalid string format", + expected: "invalid connection string format", + }, } for _, tt := range tests { @@ -80,7 +86,7 @@ func TestInitEventHouse(t *testing.T) { expected adx.Config }{ { - name: "valid configuration", + name: "valid connection", connection: "data source=https://example.kusto.windows.net;Database=testdb", expected: adx.Config{ Endpoint: "https://example.kusto.windows.net", @@ -90,7 +96,7 @@ func TestInitEventHouse(t *testing.T) { }, }, { - name: "valid configuration with timeout", + name: "connection with timeout", connection: "data source=https://example.kusto.windows.net;Database=testdb", timeout: config.Duration(60 * time.Second), expected: adx.Config{ @@ -101,7 +107,7 @@ func TestInitEventHouse(t *testing.T) { }, }, { - name: "valid connection string with all parameters", + name: "connection with all parameters", connection: "data source=https://example.com;database=mydb;table name=mytable;create tables=true;metrics grouping type=tablepermetric", expected: adx.Config{ Endpoint: "https://example.com", @@ -182,7 +188,7 @@ func TestInitEventStream(t *testing.T) { }, }, { - name: "valid connection with timeout", + name: "connection with timeout", connection: "Endpoint=sb://namespace.servicebus.windows.net/;SharedAccessKeyName=keyName;SharedAccessKey=key", timeout: config.Duration(60 * time.Second), expected: eventstream{ @@ -190,6 +196,25 @@ func TestInitEventStream(t *testing.T) { timeout: config.Duration(30 * time.Second), }, }, + { + name: "connection with partition key and message size", + connection: "endpoint=https://example.com;partitionkey=mykey;maxmessagesize=1024", + expected: eventstream{ + connectionString: "endpoint=https://example.com;partitionkey=mykey;maxmessagesize=1024", + partitionKey: "mykey", + options: azeventhubs.EventDataBatchOptions{MaxBytes: 1024}, + timeout: config.Duration(30 * time.Second), + }, + }, { + name: "case insensitive keys", + connection: "PARTITIONKEY=mykey;MaxMessageSize=1024", + expected: eventstream{ + connectionString: "PARTITIONKEY=mykey;MaxMessageSize=1024", + partitionKey: "mykey", + options: azeventhubs.EventDataBatchOptions{MaxBytes: 1024}, + timeout: config.Duration(30 * time.Second), + }, + }, } for _, tt := range tests { From 110000b1aeaebd766fdb21e6417b5b7aca7ba2fd Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Tue, 3 Jun 2025 15:49:19 +0200 Subject: [PATCH 24/28] Cleanup eventstream output --- .../outputs/microsoft_fabric/event_stream.go | 74 +++++++++---------- .../microsoft_fabric/microsoft_fabric.go | 36 ++++----- .../microsoft_fabric/microsoft_fabric_test.go | 47 +++++++----- 3 files changed, 78 insertions(+), 79 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/event_stream.go b/plugins/outputs/microsoft_fabric/event_stream.go index 5000a48019511..0a6e6b885a9ac 100644 --- a/plugins/outputs/microsoft_fabric/event_stream.go +++ b/plugins/outputs/microsoft_fabric/event_stream.go @@ -30,18 +30,46 @@ type eventstream struct { serializer telegraf.Serializer } -func (e *eventstream) Init() error { +func (e *eventstream) init() error { + // Parse the connection string by splitting it into key-value pairs + // and extract the extra keys used for plugin configuration + pairs := strings.Split(e.connectionString, ";") + for _, pair := range pairs { + // Split each pair into key and value + k, v, found := strings.Cut(pair, "=") + if !found { + return fmt.Errorf("invalid connection string format: %q", pair) + } + + // Only lowercase the keys as the values might be case sensitive + k = strings.ToLower(strings.TrimSpace(k)) + v = strings.TrimSpace(v) + + key := strings.ReplaceAll(k, " ", "") + switch key { + case "partitionkey": + e.partitionKey = v + case "maxmessagesize": + msgsize, err := strconv.ParseUint(v, 10, 64) + if err != nil { + return fmt.Errorf("invalid max message size: %w", err) + } + if msgsize > 0 { + e.options.MaxBytes = msgsize + } + } + } + + // Setup the JSON serializer serializer := &json.Serializer{ TimestampUnits: config.Duration(time.Nanosecond), TimestampFormat: time.RFC3339Nano, } if err := serializer.Init(); err != nil { - return err + return fmt.Errorf("setting up JSON serializer failed: %w", err) } e.serializer = serializer - if e.maxMessageSize > 0 { - e.options.MaxBytes = uint64(e.maxMessageSize) - } + return nil } @@ -67,10 +95,6 @@ func (e *eventstream) Close() error { return e.client.Close(ctx) } -func (e *eventstream) SetSerializer(serializer telegraf.Serializer) { - e.serializer = serializer -} - func (e *eventstream) Write(metrics []telegraf.Metric) error { // This context is only used for creating the batches which should not timeout as this is // not an I/O operation. Therefore avoid setting a timeout here. @@ -80,7 +104,6 @@ func (e *eventstream) Write(metrics []telegraf.Metric) error { batches := make(map[string]*azeventhubs.EventDataBatch) // Use a range loop with index for readability, while keeping ability to adjust the index for i, m := range metrics { - // Prepare the payload payload, err := e.serializer.Serialize(m) if err != nil { @@ -138,6 +161,7 @@ func (e *eventstream) Write(metrics []telegraf.Metric) error { } i -= 1 } + // Send the remaining batches that never exceeded the batch size for partition, batch := range batches { if batch.NumBytes() == 0 { @@ -147,35 +171,7 @@ func (e *eventstream) Write(metrics []telegraf.Metric) error { return fmt.Errorf("sending batch for partition %q failed: %w", partition, err) } } - return nil -} - -func (e *eventstream) parseconnectionString(cs string) error { - // Parse the connection string - // Split the connection string into key-value pairs - pairs := strings.Split(cs, ";") - for _, pair := range pairs { - // Split each pair into key and value - k, v, found := strings.Cut(pair, "=") - if !found { - return fmt.Errorf("invalid connection string format: %s", pair) - } - k = strings.ToLower(strings.TrimSpace(k)) - v = strings.TrimSpace(v) - - key := strings.ReplaceAll(k, " ", "") - switch key { - case "partitionkey": - e.partitionKey = v - case "maxmessagesize": - sz, err := strconv.ParseInt(v, 10, 64) - if err != nil { - return fmt.Errorf("invalid max message size: %w", err) - } - e.maxMessageSize = config.Size(sz) - } - } return nil } @@ -187,5 +183,5 @@ func (e *eventstream) send(ctx context.Context, batch *azeventhubs.EventDataBatc } func isEventstreamEndpoint(endpoint string) bool { - return strings.HasPrefix(endpoint, "Endpoint=sb") + return strings.HasPrefix(strings.ToLower(endpoint), "endpoint=sb") } diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric.go b/plugins/outputs/microsoft_fabric/microsoft_fabric.go index c257f8b2d4756..bfc0e35c425ec 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric.go @@ -16,8 +16,7 @@ import ( //go:embed sample.conf var sampleConfig string -type fabricOutput interface { - Init() error +type fabric interface { Connect() error Write(metrics []telegraf.Metric) error Close() error @@ -28,9 +27,8 @@ type MicrosoftFabric struct { Timeout config.Duration `toml:"timeout"` Log telegraf.Logger `toml:"-"` - eventhouse *eventhouse - eventstream *eventstream - activePlugin fabricOutput + eventhouse *eventhouse + output fabric } func (*MicrosoftFabric) SampleConfig() string { @@ -38,26 +36,24 @@ func (*MicrosoftFabric) SampleConfig() string { } func (m *MicrosoftFabric) Init() error { - + // Check input parameters if m.ConnectionString == "" { return errors.New("endpoint must not be empty") } + // Initialize the output fabric dependent on the type switch { case isEventstreamEndpoint(m.ConnectionString): - m.Log.Info("Detected EventStream endpoint, using EventStream output plugin") - eventstream := &eventstream{} - eventstream.connectionString = m.ConnectionString - eventstream.log = m.Log - eventstream.timeout = m.Timeout - if err := eventstream.parseconnectionString(m.ConnectionString); err != nil { - return fmt.Errorf("parsing connection string failed: %w", err) + m.Log.Debug("Detected EventStream endpoint...") + eventstream := &eventstream{ + connectionString: m.ConnectionString, + timeout: m.Timeout, + log: m.Log, } - m.eventstream = eventstream - if err := m.eventstream.Init(); err != nil { + if err := eventstream.init(); err != nil { return fmt.Errorf("initializing EventStream output failed: %w", err) } - m.activePlugin = eventstream + m.output = eventstream case isEventhouseEndpoint(strings.ToLower(m.ConnectionString)): m.Log.Info("Detected EventHouse endpoint, using EventHouse output plugin") // Setting up the AzureDataExplorer plugin initial properties @@ -72,7 +68,7 @@ func (m *MicrosoftFabric) Init() error { if err := eventhouse.parseconnectionString(m.ConnectionString); err != nil { return fmt.Errorf("parsing connection string failed: %w", err) } - m.activePlugin = m.eventhouse + m.output = m.eventhouse default: return errors.New("invalid connection string") } @@ -80,15 +76,15 @@ func (m *MicrosoftFabric) Init() error { } func (m *MicrosoftFabric) Close() error { - return m.activePlugin.Close() + return m.output.Close() } func (m *MicrosoftFabric) Connect() error { - return m.activePlugin.Connect() + return m.output.Connect() } func (m *MicrosoftFabric) Write(metrics []telegraf.Metric) error { - return m.activePlugin.Write(metrics) + return m.output.Write(metrics) } func init() { diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go index f4153dd809de1..f1efbb2926c5e 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go @@ -28,7 +28,7 @@ func TestInitFail(t *testing.T) { expected: "invalid connection string", }, { - name: "Malformed connection string", + name: "malformed connection string", connection: "endpoint=;key=;", expected: "invalid connection string", }, @@ -40,7 +40,7 @@ func TestInitFail(t *testing.T) { { name: "invalid eventhouse connection string format", connection: "invalid string format", - expected: "invalid connection string format", + expected: "invalid connection string", }, { name: "invalid eventhouse metrics grouping type", @@ -53,14 +53,14 @@ func TestInitFail(t *testing.T) { expected: "invalid setting", }, { - name: "invalid eventstream connection string", + name: "invalid eventstream connection format", connection: "Endpoint=sb://namespace.servicebus.windows.net/;invalid_param", - expected: "parsing connection string failed", + expected: "invalid connection string format", }, { - name: "invalid eventstream connection string format", - connection: "invalid string format", - expected: "invalid connection string format", + name: "invalid eventstream max message size", + connection: "Endpoint=sb://namespace.servicebus.windows.net/;maxmessagesize=-4", + expected: "invalid max message size", }, } @@ -158,15 +158,18 @@ func TestInitEventHouse(t *testing.T) { // Setup the plugin plugin := &MicrosoftFabric{ ConnectionString: tt.connection, - Timeout: tt.timeout, + Timeout: config.Duration(30 * time.Second), // default set by init() Log: testutil.Logger{}, } + if tt.timeout > 0 { + plugin.Timeout = tt.timeout + } require.NoError(t, plugin.Init()) // Check the created plugin - require.NotNil(t, plugin.activePlugin, "active plugin should have been set") - ap, ok := plugin.activePlugin.(*eventhouse) - require.Truef(t, ok, "expected evenhouse plugin but got %T", plugin.activePlugin) + require.NotNil(t, plugin.output, "active plugin should have been set") + ap, ok := plugin.output.(*eventhouse) + require.Truef(t, ok, "expected evenhouse plugin but got %T", plugin.output) require.Equal(t, tt.expected, ap.Config) }) } @@ -193,23 +196,23 @@ func TestInitEventStream(t *testing.T) { timeout: config.Duration(60 * time.Second), expected: eventstream{ connectionString: "Endpoint=sb://namespace.servicebus.windows.net/;SharedAccessKeyName=keyName;SharedAccessKey=key", - timeout: config.Duration(30 * time.Second), + timeout: config.Duration(60 * time.Second), }, }, { name: "connection with partition key and message size", - connection: "endpoint=https://example.com;partitionkey=mykey;maxmessagesize=1024", + connection: "Endpoint=sb://example.com;partitionkey=mykey;maxmessagesize=1024", expected: eventstream{ - connectionString: "endpoint=https://example.com;partitionkey=mykey;maxmessagesize=1024", + connectionString: "Endpoint=sb://example.com;partitionkey=mykey;maxmessagesize=1024", partitionKey: "mykey", options: azeventhubs.EventDataBatchOptions{MaxBytes: 1024}, timeout: config.Duration(30 * time.Second), }, }, { name: "case insensitive keys", - connection: "PARTITIONKEY=mykey;MaxMessageSize=1024", + connection: "endpoint=sb://example.com;PARTITIONKEY=mykey;MaxMessageSize=1024", expected: eventstream{ - connectionString: "PARTITIONKEY=mykey;MaxMessageSize=1024", + connectionString: "endpoint=sb://example.com;PARTITIONKEY=mykey;MaxMessageSize=1024", partitionKey: "mykey", options: azeventhubs.EventDataBatchOptions{MaxBytes: 1024}, timeout: config.Duration(30 * time.Second), @@ -222,15 +225,19 @@ func TestInitEventStream(t *testing.T) { // Setup plugin plugin := &MicrosoftFabric{ ConnectionString: tt.connection, - Timeout: tt.timeout, + Timeout: config.Duration(30 * time.Second), // default set by init() Log: testutil.Logger{}, } + if tt.timeout > 0 { + plugin.Timeout = tt.timeout + } + require.NoError(t, plugin.Init()) // Check the created plugin - require.NotNil(t, plugin.activePlugin, "active plugin should have been set") - ap, ok := plugin.activePlugin.(*eventstream) - require.Truef(t, ok, "expected evenstream plugin but got %T", plugin.activePlugin) + require.NotNil(t, plugin.output, "active plugin should have been set") + ap, ok := plugin.output.(*eventstream) + require.Truef(t, ok, "expected evenstream plugin but got %T", plugin.output) require.Equal(t, tt.expected.connectionString, ap.connectionString) require.Equal(t, tt.expected.timeout, ap.timeout) require.Equal(t, tt.expected.partitionKey, ap.partitionKey) From 6d3147253c26833b95335f20c9a6c1eadfc94731 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Tue, 3 Jun 2025 20:52:39 +0200 Subject: [PATCH 25/28] Cleanup eventhouse output --- .../outputs/microsoft_fabric/event_house.go | 119 +++++++++--------- .../microsoft_fabric/event_house_test.go | 46 +------ .../microsoft_fabric/microsoft_fabric.go | 25 ++-- .../microsoft_fabric/microsoft_fabric_test.go | 26 ++-- 4 files changed, 90 insertions(+), 126 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/event_house.go b/plugins/outputs/microsoft_fabric/event_house.go index a13ee9203f5f4..e1f002164e05e 100644 --- a/plugins/outputs/microsoft_fabric/event_house.go +++ b/plugins/outputs/microsoft_fabric/event_house.go @@ -3,6 +3,7 @@ package microsoft_fabric import ( "errors" "fmt" + "slices" "strings" "time" @@ -15,6 +16,7 @@ import ( ) type eventhouse struct { + connectionString string adx.Config client *adx.Client @@ -22,7 +24,52 @@ type eventhouse struct { serializer telegraf.Serializer } -func (e *eventhouse) Init() error { +func (e *eventhouse) init() error { + // Initialize defaults + e.CreateTables = true + + // Parse the connection string by splitting it into key-value pairs + // and extract the extra keys used for plugin configuration + pairs := strings.Split(e.connectionString, ";") + for _, pair := range pairs { + // Split each pair into key and value + k, v, found := strings.Cut(pair, "=") + if !found { + return fmt.Errorf("invalid connection string format: %s", pair) + } + + // Only lowercase the keys as the values might be case sensitive + k = strings.ToLower(strings.TrimSpace(k)) + v = strings.TrimSpace(v) + + key := strings.ReplaceAll(k, " ", "") + switch key { + case "datasource", "addr", "address", "networkaddress", "server": + e.Endpoint = v + case "initialcatalog", "database": + e.Database = v + case "ingestiontype": + e.IngestionType = v + case "tablename": + e.TableName = v + case "createtables": + switch v { + case "true": + e.CreateTables = true + case "false": + e.CreateTables = false + default: + return fmt.Errorf("invalid setting %q for %q", v, k) + } + case "metricsgroupingtype": + if v != adx.TablePerMetric && v != adx.SingleTable { + return errors.New("metrics grouping type is not valid:" + v) + } + e.MetricsGrouping = v + } + } + + // Setup the JSON serializer serializer := &json.Serializer{ TimestampUnits: config.Duration(time.Nanosecond), TimestampFormat: time.RFC3339Nano, @@ -31,30 +78,11 @@ func (e *eventhouse) Init() error { return fmt.Errorf("initializing JSON serializer failed: %w", err) } e.serializer = serializer - e.Config = adx.Config{} - e.CreateTables = true - return nil -} - -func isEventhouseEndpoint(endpoint string) bool { - prefixes := []string{ - "data source=", - "addr=", - "address=", - "network address=", - "server=", - } - for _, prefix := range prefixes { - if strings.HasPrefix(strings.ToLower(endpoint), prefix) { - return true - } - } - return false + return nil } func (e *eventhouse) Connect() error { - fmt.Printf("cfg: %+v\n", e.Config) client, err := e.NewClient("Kusto.Telegraf", e.log) if err != nil { return fmt.Errorf("creating new client failed: %w", err) @@ -119,42 +147,17 @@ func (e *eventhouse) writeSingleTable(metrics []telegraf.Metric) error { return err } -func (e *eventhouse) parseconnectionString(cs string) error { - // Parse the connection string to extract the endpoint and database - // Split the connection string into key-value pairs - pairs := strings.Split(cs, ";") - for _, pair := range pairs { - // Split each pair into key and value - k, v, found := strings.Cut(pair, "=") - if !found { - return fmt.Errorf("invalid connection string format: %s", pair) - } - k = strings.ToLower(strings.TrimSpace(k)) - v = strings.TrimSpace(v) - switch k { - case "data source", "addr", "address", "network address", "server": - e.Endpoint = v - case "initial catalog", "database": - e.Database = v - case "ingestion type", "ingestiontype": - e.IngestionType = v - case "table name", "tablename": - e.TableName = v - case "create tables", "createtables": - switch v { - case "true": - e.CreateTables = true - case "false": - e.CreateTables = false - default: - return fmt.Errorf("invalid setting %q for %q", v, k) - } - case "metrics grouping type", "metricsgroupingtype": - if v != adx.TablePerMetric && v != adx.SingleTable { - return errors.New("metrics grouping type is not valid:" + v) - } - e.MetricsGrouping = v - } +func isEventhouseEndpoint(endpoint string) bool { + prefixes := []string{ + "data source=", + "addr=", + "address=", + "network address=", + "server=", } - return nil + + ep := strings.ToLower(endpoint) + return slices.ContainsFunc(prefixes, func(prefix string) bool { + return strings.HasPrefix(ep, prefix) + }) } diff --git a/plugins/outputs/microsoft_fabric/event_house_test.go b/plugins/outputs/microsoft_fabric/event_house_test.go index d8545541ffac0..ceca2ca639598 100644 --- a/plugins/outputs/microsoft_fabric/event_house_test.go +++ b/plugins/outputs/microsoft_fabric/event_house_test.go @@ -17,7 +17,7 @@ func TestEventHouseConnectSuccess(t *testing.T) { }{ { name: "valid configuration", - endpoint: "https://example.com", + endpoint: "addr=https://example.com", database: "testdb", }, } @@ -26,13 +26,13 @@ func TestEventHouseConnectSuccess(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // Setup plugin plugin := &eventhouse{ + connectionString: tt.endpoint, Config: adx.Config{ - Endpoint: tt.endpoint, Database: tt.database, }, log: testutil.Logger{}, } - require.NoError(t, plugin.Init()) + require.NoError(t, plugin.init()) // Check for successful connection and client creation require.NoError(t, plugin.Connect()) @@ -41,46 +41,6 @@ func TestEventHouseConnectSuccess(t *testing.T) { } } -func TestEventHouseConnectFail(t *testing.T) { - tests := []struct { - name string - endpoint string - database string - expected string - }{ - { - name: "empty endpoint", - endpoint: "", - database: "testdb", - expected: "endpoint configuration cannot be empty", - }, - { - name: "empty database", - endpoint: "https://example.com", - database: "", - expected: "database configuration cannot be empty", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Setup plugin - plugin := &eventhouse{ - Config: adx.Config{ - Endpoint: tt.endpoint, - Database: tt.database, - }, - log: testutil.Logger{}, - } - require.NoError(t, plugin.Init()) - - // Connect should fail - require.ErrorContains(t, plugin.Connect(), tt.expected) - require.Nil(t, plugin.client) - }) - } -} - func TestIsEventhouseEndpoint(t *testing.T) { tests := []struct { name string diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric.go b/plugins/outputs/microsoft_fabric/microsoft_fabric.go index bfc0e35c425ec..e97b9e362697f 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric.go @@ -10,6 +10,7 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/plugins/common/adx" "github.com/influxdata/telegraf/plugins/outputs" ) @@ -27,8 +28,7 @@ type MicrosoftFabric struct { Timeout config.Duration `toml:"timeout"` Log telegraf.Logger `toml:"-"` - eventhouse *eventhouse - output fabric + output fabric } func (*MicrosoftFabric) SampleConfig() string { @@ -55,20 +55,19 @@ func (m *MicrosoftFabric) Init() error { } m.output = eventstream case isEventhouseEndpoint(strings.ToLower(m.ConnectionString)): - m.Log.Info("Detected EventHouse endpoint, using EventHouse output plugin") + m.Log.Debug("Detected EventHouse endpoint...") // Setting up the AzureDataExplorer plugin initial properties - eventhouse := &eventhouse{} - m.eventhouse = eventhouse - if err := m.eventhouse.Init(); err != nil { - return fmt.Errorf("initializing EventHouse output failed: %w", err) + eventhouse := &eventhouse{ + connectionString: m.ConnectionString, + Config: adx.Config{ + Timeout: m.Timeout, + }, + log: m.Log, } - eventhouse.Endpoint = m.ConnectionString - eventhouse.log = m.Log - eventhouse.Timeout = m.Timeout - if err := eventhouse.parseconnectionString(m.ConnectionString); err != nil { - return fmt.Errorf("parsing connection string failed: %w", err) + if err := eventhouse.init(); err != nil { + return fmt.Errorf("initializing EventHouse output failed: %w", err) } - m.output = m.eventhouse + m.output = eventhouse default: return errors.New("invalid connection string") } diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go index f1efbb2926c5e..df1849c4b5b87 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric_test.go @@ -32,15 +32,10 @@ func TestInitFail(t *testing.T) { connection: "endpoint=;key=;", expected: "invalid connection string", }, - { - name: "invalid eventhouse connection string", - connection: "data source=https://example.kusto.windows.net;invalid_param", - expected: "parsing connection string failed", - }, { name: "invalid eventhouse connection string format", - connection: "invalid string format", - expected: "invalid connection string", + connection: "data source=https://example.kusto.windows.net;invalid_param", + expected: "invalid connection string format", }, { name: "invalid eventhouse metrics grouping type", @@ -113,24 +108,29 @@ func TestInitEventHouse(t *testing.T) { Endpoint: "https://example.com", Database: "mydb", TableName: "mytable", - CreateTables: true, MetricsGrouping: "tablepermetric", + CreateTables: true, + Timeout: config.Duration(30 * time.Second), }, }, { name: "case insensitive parameters", connection: "DATA SOURCE=https://example.com;DATABASE=mydb", expected: adx.Config{ - Endpoint: "https://example.com", - Database: "mydb", + Endpoint: "https://example.com", + Database: "mydb", + CreateTables: true, + Timeout: config.Duration(30 * time.Second), }, }, { name: "server parameter instead of data source", connection: "server=https://example.com;database=mydb", expected: adx.Config{ - Endpoint: "https://example.com", - Database: "mydb", + Endpoint: "https://example.com", + Database: "mydb", + CreateTables: true, + Timeout: config.Duration(30 * time.Second), }, }, { @@ -140,6 +140,7 @@ func TestInitEventHouse(t *testing.T) { Endpoint: "https://example.com", Database: "mydb", CreateTables: true, + Timeout: config.Duration(30 * time.Second), }, }, { @@ -149,6 +150,7 @@ func TestInitEventHouse(t *testing.T) { Endpoint: "https://example.com", Database: "mydb", CreateTables: false, + Timeout: config.Duration(30 * time.Second), }, }, } From 45c01c9ecde87aafdf27bb9cf6bebdcfa89833c7 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Tue, 3 Jun 2025 21:13:35 +0200 Subject: [PATCH 26/28] Cleanup documentation --- plugins/outputs/microsoft_fabric/README.md | 162 +++++++++--------- .../microsoft_fabric/microsoft_fabric.go | 1 - plugins/outputs/microsoft_fabric/sample.conf | 2 +- 3 files changed, 81 insertions(+), 84 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/README.md b/plugins/outputs/microsoft_fabric/README.md index 3b02edf2179fd..692fcad2f3d54 100644 --- a/plugins/outputs/microsoft_fabric/README.md +++ b/plugins/outputs/microsoft_fabric/README.md @@ -23,7 +23,7 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. # Sends metrics to Microsoft Fabric [[outputs.microsoft_fabric]] ## The URI property of the resource on Azure - connection_string = "https://trd-abcd.xx.kusto.fabric.microsoft.com;Database=kusto_eh; Table Name=telegraf_dump;Key=value" + connection_string = "https://trd-abcd.xx.kusto.fabric.microsoft.com;Database=kusto_eh;Table Name=telegraf_dump;Key=value" ## Client timeout # timeout = "30s" @@ -31,103 +31,102 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. ### Connection String -The `connection_string` provide the information necessary for a client application to +The `connection_string` provide information necessary for the plugin to establish a connection to the Fabric service endpoint. It is a -semicolon-delimited list of name-value parameter pairs, optionally prefixed by a -single URI. The `connection_string` setting is specific to the type of endpoint you are using. -The sections below will detail on the required and available name-value pairs for -each type. +semicolon-delimited list of name-value parameter pairs, optionally prefixed by +a single URI. The setting is specific to the type of endpoint you are using. +The sections below will detail on the required and available name-value pairs +for each type. ### EventHouse -This plugin allows you to leverage Microsoft Fabric's capabilities -to store and analyze your Telegraf metrics. Eventhouse is a high-performance, -scalable data store designed for real-time analytics. It allows you to ingest, -store, and query large volumes of data with low latency. For more information, -visit the [Eventhouse documentation][eventhousedocs]. +This plugin allows you to leverage Microsoft Fabric's capabilities to store and +analyze your Telegraf metrics. Eventhouse is a high-performance, scalable +data-store designed for real-time analytics. It allows you to ingest, store and +query large volumes of data with low latency. For more information, visit the +[Eventhouse documentation][eventhousedocs]. + +The following table lists all the possible properties that can be included in a +connection string and provide alias names for each property. + +| Property name | Aliases | Description | +|---|---|---| +| Client Version for Tracing | | The property used when tracing the client version. | +| Data Source | Addr, Address, Network Address, Server | The URI specifying the Kusto service endpoint. For example, `https://mycluster.fabric.windows.net`. | +| Initial Catalog | Database | The default database name. For example, `MyDatabase`. | +| Ingestion Type | IngestionType | Values can be set to `managed` for streaming ingestion with fallback to batched ingestion or the `queued` method for queuing up metrics and process sequentially | +| Table Name | TableName | Name of the single table to store all the metrics; only needed if `metrics_grouping_type` is `singletable` | +| Create Tables | CreateTables | Creates tables and relevant mapping if `true` (default). Otherwise table and mapping creation is skipped. This is useful for running Telegraf with the lowest possible permissions i.e. table ingestor role. | +| Metrics Grouping Type | MetricsGroupingType | Type of metrics grouping used when pushing to Eventhouse either being `tablepermetric` or `singletable`. Default is "tablepermetric" for one table per different metric.| [eventhousedocs]: https://learn.microsoft.com/fabric/real-time-intelligence/eventhouse -The following table lists all the possible properties that can be included in a -connection string and provide alias names for each property: +#### Metrics Grouping -The following table lists all the possible properties that can be included in a -connection string and provide alias names for each property. +Metrics can be grouped in two ways to be sent to Azure Data Explorer. To specify +which metric grouping type the plugin should use, the respective value should be +given to the `Metrics Grouping Type` in the connection string. If no value is +given, by default, the metrics will be grouped using `tablepermetric`. + +#### TablePerMetric -| Property name | Description | -|---|---| -| Client Version for Tracing | The property used when tracing the client version. | -| Data Source

**Aliases:** Addr, Address, Network Address, Server | The URI specifying the Kusto service endpoint. For example, `https://mycluster.fabric.windows.net`. | -| Initial Catalog

**Alias:** Database | The default database name. For example, `MyDatabase`. | -| Ingestion Type

**Alias:** IngestionType | Values can be set to,
- managed : Streaming ingestion with fallback to batched ingestion or the "queued" method below
- queued : Queue up metrics data and process sequentially | -| Table Name

**Alias:** TableName | Name of the single table to store all the metrics (Only needed if metrics_grouping_type is "SingleTable") | -| Create Tables

**Alias:** CreateTables | Creates tables and relevant mapping if set to true(default).
Skips table and mapping creation if set to false, this is useful for running Telegraf with the lowest possible permissions i.e. table ingestor role. | -| Metrics Grouping Type

**Alias:** MetricsGroupingType | Type of metrics grouping used when pushing to Eventhouse. values can be set, 'tablepermetric' and 'singletable'. Default is "tablepermetric" for one table per different metric.| - -* *Metrics Grouping* - - Metrics can be grouped in two ways to be sent to Azure Data Explorer. To specify - which metric grouping type the plugin should use, the respective value should be - given to the `Metrics Grouping Type` in the connection string. If no value is given, by default, the metrics will be grouped using - `tablepermetric`. - -* *TablePerMetric* - - The plugin will group the metrics by the metric name, and will send each group - of metrics to an Azure Data Explorer table. If the table doesn't exist the - plugin will create the table, if the table exists then the plugin will try to - merge the Telegraf metric schema to the existing table. For more information +The plugin will group the metrics by the metric name and will send each group +of metrics to an Azure Data Explorer table. If the table doesn't exist the +plugin will create the table, if the table exists then the plugin will try to +merge the Telegraf metric schema to the existing table. For more information about the merge process check the [`.create-merge` documentation][create-merge]. - The table name will match the `name` property of the metric, this means that the - name of the metric should comply with the Azure Data Explorer table naming - constraints in case you plan to add a prefix to the metric name. +The table name will match the metric name, i.e. the name of the metric must +comply with the Azure Data Explorer table naming constraints in case you plan +to add a prefix to the metric name. [create-merge]: https://docs.microsoft.com/en-us/azure/data-explorer/kusto/management/create-merge-table-command -* *SingleTable* +#### SingleTable - The plugin will send all the metrics received to a single Azure Data Explorer - table. The name of the table must be supplied via `table_name` in the config - file. If the table doesn't exist the plugin will create the table, if the table - exists then the plugin will try to merge the Telegraf metric schema to the - existing table. For more information about the merge process check the - [`.create-merge` documentation][create-merge]. +The plugin will send all the metrics received to a single Azure Data Explorer +table. The name of the table must be supplied via `table_name` parameter in the +`connection_string`. If the table doesn't exist the plugin will create the +table, if the table exists then the plugin will try to merge the Telegraf metric +schema to the existing table. For more information about the merge process check +the [`.create-merge` documentation][create-merge]. -* *Tables Schema* +#### Tables Schema - The schema of the Eventhouse table will match the structure of the - Telegraf `Metric` object. The corresponding command - generated by the plugin would be like the following: +The schema of the Eventhouse table will match the structure of the metric. +The corresponding command generated by the plugin would be like the following: - ```kql - .create-merge table ['table-name'] (['fields']:dynamic, ['name']:string, ['tags']:dynamic, ['timestamp']:datetime) - ``` +```kql +.create-merge table ['table-name'] (['fields']:dynamic, ['name']:string, ['tags']:dynamic, ['timestamp']:datetime) +``` + +The corresponding table mapping would be like the following: - The corresponding table mapping would be like the following: +```kql +.create-or-alter table ['table-name'] ingestion json mapping 'table-name_mapping' '[{"column":"fields", "Properties":{"Path":"$[\'fields\']"}},{"column":"name", "Properties":{"Path":"$[\'name\']"}},{"column":"tags", "Properties":{"Path":"$[\'tags\']"}},{"column":"timestamp", "Properties":{"Path":"$[\'timestamp\']"}}]' +``` - ```kql - .create-or-alter table ['table-name'] ingestion json mapping 'table-name_mapping' '[{"column":"fields", "Properties":{"Path":"$[\'fields\']"}},{"column":"name", "Properties":{"Path":"$[\'name\']"}},{"column":"tags", "Properties":{"Path":"$[\'tags\']"}},{"column":"timestamp", "Properties":{"Path":"$[\'timestamp\']"}}]' - ``` +> [!NOTE] +> This plugin will automatically create tables and corresponding table mapping +> using the command above. - **Note**: This plugin will automatically create Eventhouse tables and - corresponding table mapping as per the above mentioned commands. +#### Ingestion type -* *Ingestion type* +> [!NOTE] +> [Streaming ingestion][streaming] has to be enabled on ADX in case of +> `managed` operation. - **Note**: - [Streaming ingestion](https://aka.ms/AAhlg6s) - has to be enabled on ADX [configure the ADX cluster] - in case of `managed` option. - Refer the query below to check if streaming is enabled +Refer to the following query below to check if streaming is enabled: - ```kql - .show database policy streamingingestion - ``` +```kql +.show database policy streamingingestion +``` - To know more about configuration, supported authentication methods and querying ingested data, read the [documentation][ethdocs] +To learn more about configuration, supported authentication methods and querying +ingested data, check the [documentation][ethdocs]. - [ethdocs]: https://learn.microsoft.com/azure/data-explorer/ingest-data-telegraf +[streaming]: https://learn.microsoft.com/en-us/azure/data-explorer/ingest-data-streaming?tabs=azure-portal%2Ccsharp +[ethdocs]: https://learn.microsoft.com/azure/data-explorer/ingest-data-telegraf ### Eventstream @@ -135,16 +134,15 @@ Eventstreams allow you to bring real-time events into Fabric, transform them, and then route them to various destinations without writing any code (no-code). For more information, visit the [Eventstream documentation][eventstream_docs]. -To communicate with an eventstream, you need a connection string for the -namespace or the event hub. If you use a connection string to the namespace -from your application, following are the properties that can be added -to the standard [Eventstream connection string][ecs] like a key value pair. +To communicate with an eventstream, you need to specify a connection string for +the namespace or the event hub. The following properties can be added to the +standard [Eventstream connection string][ecs] using key-value pairs: -| Property name | Description | -|---|---| -| Partition Key

**Aliases:** PartitionKey | Partition key to use for the event Metric tag or field name to use for the event partition key. The value of this tag or field is set as the key for events if it exists. If both, tag and field, exist the tag is preferred. | -| Max Message Size

**Aliases:** MaxMessageSize | Set the maximum batch message size in bytes The allowable size depends on the Event Hub tier, see for details. If unset the default size defined by Azure Event Hubs is used (currently 1,000,000 bytes) | - -[ecs]: https://learn.microsoft.com/azure/event-hubs/event-hubs-get-connection-string +| Property name | Aliases | Description | +|---|---|---| +| Partition Key | PartitionKey | Metric tag or field name to use for the event partition key if it exists. If both, tag and field, exist the tag is takes precedence, otherwise the value `` is used | +| Max Message Size| MaxMessageSize | Maximum batch message size in bytes The allowable size depends on the Event Hub tier, see [tier information][tiers] for details. If unset the default size defined by Azure Event Hubs is used (currently 1,000,000 bytes) | [eventstream_docs]: https://learn.microsoft.com/fabric/real-time-intelligence/event-streams/overview?tabs=enhancedcapabilities +[ecs]: https://learn.microsoft.com/azure/event-hubs/event-hubs-get-connection-string +[tiers]: https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas#basic-vs-standard-vs-premium-vs-dedicated-tiers diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric.go b/plugins/outputs/microsoft_fabric/microsoft_fabric.go index e97b9e362697f..7579ed4291f8d 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric.go @@ -56,7 +56,6 @@ func (m *MicrosoftFabric) Init() error { m.output = eventstream case isEventhouseEndpoint(strings.ToLower(m.ConnectionString)): m.Log.Debug("Detected EventHouse endpoint...") - // Setting up the AzureDataExplorer plugin initial properties eventhouse := &eventhouse{ connectionString: m.ConnectionString, Config: adx.Config{ diff --git a/plugins/outputs/microsoft_fabric/sample.conf b/plugins/outputs/microsoft_fabric/sample.conf index 4f935189c99ac..6707bcc74c277 100644 --- a/plugins/outputs/microsoft_fabric/sample.conf +++ b/plugins/outputs/microsoft_fabric/sample.conf @@ -1,7 +1,7 @@ # Sends metrics to Microsoft Fabric [[outputs.microsoft_fabric]] ## The URI property of the resource on Azure - connection_string = "https://trd-abcd.xx.kusto.fabric.microsoft.com;Database=kusto_eh; Table Name=telegraf_dump;Key=value" + connection_string = "https://trd-abcd.xx.kusto.fabric.microsoft.com;Database=kusto_eh;Table Name=telegraf_dump;Key=value" ## Client timeout # timeout = "30s" From e8eb8022be0eb1e7017067e6798fa8d0961e67a2 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Tue, 3 Jun 2025 21:18:19 +0200 Subject: [PATCH 27/28] Fix batching iteration --- plugins/outputs/microsoft_fabric/event_stream.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/outputs/microsoft_fabric/event_stream.go b/plugins/outputs/microsoft_fabric/event_stream.go index 0a6e6b885a9ac..7d92ca3c49b96 100644 --- a/plugins/outputs/microsoft_fabric/event_stream.go +++ b/plugins/outputs/microsoft_fabric/event_stream.go @@ -100,10 +100,12 @@ func (e *eventstream) Write(metrics []telegraf.Metric) error { // not an I/O operation. Therefore avoid setting a timeout here. ctx := context.Background() + // Iterate over the metrics and group them to batches batchOptions := e.options batches := make(map[string]*azeventhubs.EventDataBatch) - // Use a range loop with index for readability, while keeping ability to adjust the index - for i, m := range metrics { + for i := 0; i < len(metrics); i++ { + m := metrics[i] + // Prepare the payload payload, err := e.serializer.Serialize(m) if err != nil { @@ -159,7 +161,7 @@ func (e *eventstream) Write(metrics []telegraf.Metric) error { if err != nil { return fmt.Errorf("creating batch for partition %q failed: %w", partition, err) } - i -= 1 + i-- } // Send the remaining batches that never exceeded the batch size From 6be4d7dd8016de69bff2040c35edb29f526abce0 Mon Sep 17 00:00:00 2001 From: asaharn Date: Thu, 12 Jun 2025 13:51:39 +0530 Subject: [PATCH 28/28] Added few changes as per second review cycle --- plugins/outputs/microsoft_fabric/event_house.go | 4 ++++ plugins/outputs/microsoft_fabric/event_house_test.go | 2 ++ plugins/outputs/microsoft_fabric/event_stream.go | 8 ++++++++ plugins/outputs/microsoft_fabric/microsoft_fabric.go | 2 +- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/plugins/outputs/microsoft_fabric/event_house.go b/plugins/outputs/microsoft_fabric/event_house.go index e1f002164e05e..4acb39e7406a7 100644 --- a/plugins/outputs/microsoft_fabric/event_house.go +++ b/plugins/outputs/microsoft_fabric/event_house.go @@ -32,6 +32,10 @@ func (e *eventhouse) init() error { // and extract the extra keys used for plugin configuration pairs := strings.Split(e.connectionString, ";") for _, pair := range pairs { + // Skip empty pairs + if strings.TrimSpace(pair) == "" { + continue + } // Split each pair into key and value k, v, found := strings.Cut(pair, "=") if !found { diff --git a/plugins/outputs/microsoft_fabric/event_house_test.go b/plugins/outputs/microsoft_fabric/event_house_test.go index ceca2ca639598..2d5f3277af58d 100644 --- a/plugins/outputs/microsoft_fabric/event_house_test.go +++ b/plugins/outputs/microsoft_fabric/event_house_test.go @@ -37,6 +37,8 @@ func TestEventHouseConnectSuccess(t *testing.T) { // Check for successful connection and client creation require.NoError(t, plugin.Connect()) require.NotNil(t, plugin.client) + // Clean up resources + require.NoError(t, plugin.Close()) }) } } diff --git a/plugins/outputs/microsoft_fabric/event_stream.go b/plugins/outputs/microsoft_fabric/event_stream.go index 7d92ca3c49b96..74e104f6f867d 100644 --- a/plugins/outputs/microsoft_fabric/event_stream.go +++ b/plugins/outputs/microsoft_fabric/event_stream.go @@ -35,6 +35,10 @@ func (e *eventstream) init() error { // and extract the extra keys used for plugin configuration pairs := strings.Split(e.connectionString, ";") for _, pair := range pairs { + // Skip empty pairs + if strings.TrimSpace(pair) == "" { + continue + } // Split each pair into key and value k, v, found := strings.Cut(pair, "=") if !found { @@ -89,6 +93,10 @@ func (e *eventstream) Connect() error { } func (e *eventstream) Close() error { + if e.client == nil { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(e.timeout)) defer cancel() diff --git a/plugins/outputs/microsoft_fabric/microsoft_fabric.go b/plugins/outputs/microsoft_fabric/microsoft_fabric.go index 7579ed4291f8d..ebf5b5762a074 100644 --- a/plugins/outputs/microsoft_fabric/microsoft_fabric.go +++ b/plugins/outputs/microsoft_fabric/microsoft_fabric.go @@ -68,7 +68,7 @@ func (m *MicrosoftFabric) Init() error { } m.output = eventhouse default: - return errors.New("invalid connection string") + return errors.New("invalid connection string: unable to detect endpoint type (EventStream or EventHouse)") } return nil }