azure blob storage upload support

.snyk

@@ -33,7 +33,7 @@ ignore:
   SNYK-GOLANG-GOLANGORGXIMAGETIFF-7268348:
     - '*':
         reason: No fix available yet
-        expires: 2025-01-11T00:00:00.000Z
+        expires: 2025-03-01T00:00:00.000Z
         created: 2024-07-11T23:07:11.176Z
   SNYK-GOLANG-K8SIOCLIENTGOTRANSPORT-7538822:
     - '*':
@@ -64,18 +64,23 @@ ignore:
           volume/volumeMount to their deployment
         expires: 2200-01-01T00:00:00.000Z
         created: 2024-07-23T00:43:23.074Z
+  SNYK-GOLANG-K8SIOAPIMACHINERYPKGUTILRUNTIME-8367153:
+    - '*':
+        reason: >-
+          When bumping this dependency to the recommended version several
+          critical vulnerabilities are introduced. Ignoring for 60 days.
+        expires: 2025-02-13T00:00:00.000Z
+        created: 2024-11-13T23:09:26.278Z
   SNYK-ALPINE320-OPENSSL-8235201:
     - '*':
         reason: >-
           Alpine 3.20.3 is the newest version of alpine available, ignoring
           temporarily until patch version is ready
         expires: 2024-11-30T00:00:00.000Z
         created: 2024-10-30T18:01:47.217Z
-  SNYK-GOLANG-K8SIOAPIMACHINERYPKGUTILRUNTIME-8367153:
+  SNYK-GOLANG-GITHUBCOMGOLANGJWTJWT-8341243:
     - '*':
-        reason: >-
-          Ignoring vulnerability for 30 days as newer versions of k8s.io
-          contain several critical vulnerabilities and are still in alpha
-        expires: 2025-01-01T00:00:00.000Z
-        created: 2024-11-13T23:30:00.999Z
+        reason: No fix available
+        expires: 2025-03-01T00:00:00.000Z
+        created: 2025-01-16T21:14:33.951Z
 patch: {}

README.md

@@ -39,7 +39,7 @@ Cloudability Metrics Agent currently does not support Rancher or On Prem cluster
 
 | Environment Variable                           |                                                                                             Description                                                                                              |
 |------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|
-| CLOUDABILITY_API_KEY                           |                                                                                    Required: Cloudability api key                                                                                    |
+| CLOUDABILITY_API_KEY                           |                                            Required/Optional: Cloudability api key. Optional only if using Custom S3 upload or Custom Azure blob upload.                                             |
 | CLOUDABILITY_CLUSTER_NAME                      |                                                            Required: The cluster name to be used for the cluster the agent is running in.                                                            |
 | CLOUDABILITY_POLL_INTERVAL                     |                                                                    Optional: The interval (Seconds) to poll metrics. Default: 180                                                                    |
 | CLOUDABILITY_OUTBOUND_PROXY                    |                    Optional: The URL of an outbound HTTP/HTTPS proxy for the agent to use (eg: http://x.x.x.x:8080). The URL must contain the scheme prefix (http:// or https://)                    |
@@ -59,6 +59,11 @@ Cloudability Metrics Agent currently does not support Rancher or On Prem cluster
 | CLOUDABILITY_UPLOAD_REGION                     |          Optional: The region the metrics-agent will upload data to. Default `us-west-2`. Supported values: `us-west-2`, `eu-central-1`, `ap-southeast-2`, `me-central-1`, `us-gov-west-1`           |
 | CLOUDABILITY_CUSTOM_S3_BUCKET                  |  Optional: A custom S3 bucket the metrics-agent will upload data to. If set, the metrics-agent will ONLY upload to this custom location. CLOUDABILITY_CUSTOM_S3_REGION is REQUIRED if this is set.   |
 | CLOUDABILITY_CUSTOM_S3_REGION                  |            Optional: The AWS region that the custom s3 bucket is in. This will initialize the correct region for the s3 client. CLOUDABILITY_CUSTOM_S3_BUCKET is REQUIRED if this is set.            |
+| CLOUDABILITY_CUSTOM_AZURE_BLOB_CONTAINER_NAME  |                       Optional: A custom Azure blob container name the metrics-agent will upload data to. If set, the metrics-agent will ONLY upload to this custom location.                        |
+| CLOUDABILITY_CUSTOM_AZURE_BLOB_URL             |                                                     Optional: The Azure storage account's URL. This will be used to initialize the Azure client.                                                     |
+| CLOUDABILITY_CUSTOM_AZURE_TENANT_ID            |                       Optional: The Azure Service account's Tenant ID. This will be used to initialize the Azure client. All CUSTOM_AZURE ENV vars are REQUIRED if this is set                       |
+| CLOUDABILITY_CUSTOM_AZURE_CLIENT_ID            |                       Optional: The Azure Service account's Client ID. This will be used to initialize the Azure client. All CUSTOM_AZURE ENV vars are REQUIRED if this is set                       |
+| CLOUDABILITY_CUSTOM_AZURE_CLIENT_SECRET        |                     Optional: The Azure Service account's Client Secret. This will be used to initialize the Azure client. All CUSTOM_AZURE ENV vars are REQUIRED if this is set                     |
 
 ```sh
 
@@ -90,6 +95,11 @@ Flags:
       --upload_region                            The region the metrics-agent will upload data to. (default `us-west-2`)
       --custom_s3_bucket string                  A custom S3 bucket the metrics-agent will upload data to. - Optional
       --custom_s3_region                         The AWS region that the custom s3 bucket is created.
+      --custom_azure_blob_container_name string  A custom Azure blob container name the metrics-agent will upload data to. - Optional
+      --custom_azure_blob_url                    A custom Azure blob url metrics-agent uses to create the Azure client. - Optional
+      --custom_azure_tenant_id                   A custom Azure Service principal tenant id the metrics-agent uses to create the Azure client. - Optional
+      --custom_azure_client_id                   A custom Azure Service Principal client id the metrics-agent uses to create the Azure client. - Optional
+      --custom_azure_client_secret               A custom Azure Service Principal client secret the metrics-agent uses to create the Azure client. - Optional
 Global Flags:
       --log_format string   Format for log output (JSON,PLAIN) (default "PLAIN")
       --log_level string    Log level to run the agent at (INFO,WARN,DEBUG) (default "INFO")

charts/metrics-agent/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 2.11.36
+version: 2.11.37
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 2.11.36
+appVersion: 2.11.37

charts/metrics-agent/values.yaml

@@ -24,7 +24,7 @@ pollInterval: 180
 
 image:
   name: cloudability/metrics-agent
-  tag: 2.11.36
+  tag: 2.11.37
   pullPolicy: Always
 
 imagePullSecrets: []

cmd/kubernetesCmd.go

@@ -150,6 +150,36 @@ func init() {
 		"",
 		"The AWS region that the custom s3 bucket is in",
 	)
+	kubernetesCmd.PersistentFlags().StringVar(
+		&config.CustomAzureUploadBlobContainerName,
+		"custom_azure_blob_container_name",
+		"",
+		"The Azure blob container name the metrics-agent will upload data to.",
+	)
+	kubernetesCmd.PersistentFlags().StringVar(
+		&config.CustomAzureBlobURL,
+		"custom_azure_blob_url",
+		"",
+		"The Azure blob url the metrics-agent will upload data to.",
+	)
+	kubernetesCmd.PersistentFlags().StringVar(
+		&config.CustomAzureTenantID,
+		"custom_azure_tenant_id",
+		"",
+		"The Azure tenant id the metrics-agent uses when connecting to the azure blob.",
+	)
+	kubernetesCmd.PersistentFlags().StringVar(
+		&config.CustomAzureClientID,
+		"custom_azure_client_id",
+		"",
+		"The Azure client id the metrics-agent uses when connecting to the azure blob.",
+	)
+	kubernetesCmd.PersistentFlags().StringVar(
+		&config.CustomAzureClientSecret,
+		"custom_azure_client_secret",
+		"",
+		"The Azure client secret the metrics-agent uses when connecting to the azure blob.",
+	)
 
 	//nolint gas
 	_ = viper.BindPFlag("api_key", kubernetesCmd.PersistentFlags().Lookup("api_key"))
@@ -177,32 +207,43 @@ func init() {
 	_ = viper.BindPFlag("upload_region", kubernetesCmd.PersistentFlags().Lookup("upload_region"))
 	_ = viper.BindPFlag("custom_s3_bucket", kubernetesCmd.PersistentFlags().Lookup("custom_s3_bucket"))
 	_ = viper.BindPFlag("custom_s3_region", kubernetesCmd.PersistentFlags().Lookup("custom_s3_region"))
+	_ = viper.BindPFlag("custom_azure_blob_container_name", kubernetesCmd.PersistentFlags().Lookup(
+		"custom_azure_blob_container_name"))
+	_ = viper.BindPFlag("custom_azure_blob_url", kubernetesCmd.PersistentFlags().Lookup("custom_azure_blob_url"))
+	_ = viper.BindPFlag("custom_azure_tenant_id", kubernetesCmd.PersistentFlags().Lookup("custom_azure_tenant_id"))
+	_ = viper.BindPFlag("custom_azure_client_id", kubernetesCmd.PersistentFlags().Lookup("custom_azure_client_id"))
+	_ = viper.BindPFlag("custom_azure_client_secret", kubernetesCmd.PersistentFlags().Lookup("custom_azure_client_secret"))
 	viper.SetEnvPrefix("cloudability")
 	viper.AutomaticEnv()
 
 	RootCmd.AddCommand(kubernetesCmd)
 
 	config = kubernetes.KubeAgentConfig{
-		APIKey:                 viper.GetString("api_key"),
-		ClusterName:            viper.GetString("cluster_name"),
-		PollInterval:           viper.GetInt("poll_interval"),
-		CollectionRetryLimit:   viper.GetUint("collection_retry_limit"),
-		OutboundProxy:          viper.GetString("outbound_proxy"),
-		OutboundProxyAuth:      viper.GetString("outbound_proxy_auth"),
-		OutboundProxyInsecure:  viper.GetBool("outbound_proxy_insecure"),
-		Insecure:               viper.GetBool("insecure"),
-		Cert:                   viper.GetString("certificate_file"),
-		Key:                    viper.GetString("key_file"),
-		ConcurrentPollers:      viper.GetInt("number_of_concurrent_node_pollers"),
-		ForceKubeProxy:         viper.GetBool("force_kube_proxy"),
-		Namespace:              viper.GetString("namespace"),
-		ScratchDir:             viper.GetString("scratch_dir"),
-		InformerResyncInterval: viper.GetInt("informer_resync_interval"),
-		ParseMetricData:        viper.GetBool("parse_metric_data"),
-		HTTPSTimeout:           viper.GetInt("https_client_timeout"),
-		UploadRegion:           viper.GetString("upload_region"),
-		CustomS3UploadBucket:   viper.GetString("custom_s3_bucket"),
-		CustomS3Region:         viper.GetString("custom_s3_region"),
+		APIKey:                             viper.GetString("api_key"),
+		ClusterName:                        viper.GetString("cluster_name"),
+		PollInterval:                       viper.GetInt("poll_interval"),
+		CollectionRetryLimit:               viper.GetUint("collection_retry_limit"),
+		OutboundProxy:                      viper.GetString("outbound_proxy"),
+		OutboundProxyAuth:                  viper.GetString("outbound_proxy_auth"),
+		OutboundProxyInsecure:              viper.GetBool("outbound_proxy_insecure"),
+		Insecure:                           viper.GetBool("insecure"),
+		Cert:                               viper.GetString("certificate_file"),
+		Key:                                viper.GetString("key_file"),
+		ConcurrentPollers:                  viper.GetInt("number_of_concurrent_node_pollers"),
+		ForceKubeProxy:                     viper.GetBool("force_kube_proxy"),
+		Namespace:                          viper.GetString("namespace"),
+		ScratchDir:                         viper.GetString("scratch_dir"),
+		InformerResyncInterval:             viper.GetInt("informer_resync_interval"),
+		ParseMetricData:                    viper.GetBool("parse_metric_data"),
+		HTTPSTimeout:                       viper.GetInt("https_client_timeout"),
+		UploadRegion:                       viper.GetString("upload_region"),
+		CustomS3UploadBucket:               viper.GetString("custom_s3_bucket"),
+		CustomS3Region:                     viper.GetString("custom_s3_region"),
+		CustomAzureUploadBlobContainerName: viper.GetString("custom_azure_blob_container_name"),
+		CustomAzureBlobURL:                 viper.GetString("custom_azure_blob_url"),
+		CustomAzureTenantID:                viper.GetString("custom_azure_tenant_id"),
+		CustomAzureClientID:                viper.GetString("custom_azure_client_id"),
+		CustomAzureClientSecret:            viper.GetString("custom_azure_client_secret"),
 	}
 
 }

go.mod

@@ -3,6 +3,9 @@ module github.com/cloudability/metrics-agent
 go 1.22
 
 require (
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0
 	github.com/aws/aws-sdk-go v1.40.27
 	github.com/google/cadvisor v0.48.1
 	github.com/googleapis/gnostic v0.5.5
@@ -21,6 +24,8 @@ require (
 )
 
 require (
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/BurntSushi/toml v1.2.1 // indirect
 	github.com/alessio/shellescape v1.4.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -33,18 +38,20 @@ require (
 	github.com/go-openapi/jsonreference v0.20.1 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
@@ -56,14 +63,16 @@ require (
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.4.1 // indirect
-	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
 	golang.org/x/oauth2 v0.7.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
 	golang.org/x/term v0.27.0 // indirect
@@ -94,7 +103,7 @@ replace (
 	github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.14
 	golang.org/x/crypto => golang.org/x/crypto v0.31.0
 	golang.org/x/image => golang.org/x/image v0.10.0
-	golang.org/x/net => golang.org/x/net v0.23.0
+	golang.org/x/net => golang.org/x/net v0.33.0
 	google.golang.org/grpc => google.golang.org/grpc v1.56.3
 	google.golang.org/protobuf => google.golang.org/protobuf v1.33.0
 )