Add sinceVersion to bundle schema

.github/workflows/push.yml

@@ -318,6 +318,7 @@ jobs:
             a.addKeyword('deprecationMessage');
             a.addKeyword('doNotSuggest');
             a.addKeyword('markdownDescription');
+            a.addKeyword('sinceVersion');
             a.addKeyword('x-databricks-preview');
           }" >> keywords.js
 

bundle/internal/annotation/descriptor.go

@@ -10,6 +10,7 @@ type Descriptor struct {
 	DeprecationMessage  string `json:"deprecation_message,omitempty"`
 	Preview             string `json:"x-databricks-preview,omitempty"`
 	OutputOnly          *bool  `json:"x-databricks-field-behaviors_output_only,omitempty"`
+	SinceVersion        string `json:"sinceVersion,omitempty"`
 }
 
 const Placeholder = "PLACEHOLDER"

bundle/internal/annotation/file.go

@@ -3,11 +3,15 @@ package annotation
 import (
 	"bytes"
 	"os"
+	"slices"
+
+	yaml3 "gopkg.in/yaml.v3"
 
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/dyn/convert"
 	"github.com/databricks/cli/libs/dyn/merge"
 	"github.com/databricks/cli/libs/dyn/yamlloader"
+	"github.com/databricks/cli/libs/dyn/yamlsaver"
 )
 
 // Parsed file with annotations, expected format:
@@ -17,6 +21,31 @@ import (
 //	   description: "Description"
 type File map[string]map[string]Descriptor
 
+// Load loads annotations from a single file.
+func Load(path string) (File, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return make(File), nil
+		}
+		return nil, err
+	}
+
+	dynVal, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b))
+	if err != nil {
+		return nil, err
+	}
+
+	var data File
+	if err := convert.ToTyped(&data, dynVal); err != nil {
+		return nil, err
+	}
+	if data == nil {
+		data = make(File)
+	}
+	return data, nil
+}
+
 func LoadAndMerge(sources []string) (File, error) {
 	prev := dyn.NilValue
 	for _, path := range sources {
@@ -42,3 +71,44 @@ func LoadAndMerge(sources []string) (File, error) {
 	}
 	return data, nil
 }
+
+// Save saves the annotation file to the given path.
+func (f File) Save(path string) error {
+	annotationOrder := yamlsaver.NewOrder([]string{"description", "markdown_description", "title", "default", "enum", "since_version"})
+	style := map[string]yaml3.Style{}
+
+	order := alphabeticalOrder(f)
+	dynMap := map[string]dyn.Value{}
+	for k, v := range f {
+		style[k] = yaml3.LiteralStyle
+
+		properties := map[string]dyn.Value{}
+		propertiesOrder := alphabeticalOrder(v)
+		for key, value := range v {
+			d, err := convert.FromTyped(value, dyn.NilValue)
+			if d.Kind() == dyn.KindNil || err != nil {
+				properties[key] = dyn.NewValue(map[string]dyn.Value{}, []dyn.Location{{Line: propertiesOrder.Get(key)}})
+				continue
+			}
+			val, err := yamlsaver.ConvertToMapValue(value, annotationOrder, []string{}, map[string]dyn.Value{})
+			if err != nil {
+				return err
+			}
+			properties[key] = val.WithLocations([]dyn.Location{{Line: propertiesOrder.Get(key)}})
+		}
+
+		dynMap[k] = dyn.NewValue(properties, []dyn.Location{{Line: order.Get(k)}})
+	}
+
+	saver := yamlsaver.NewSaverWithStyle(style)
+	return saver.SaveAsYAML(dynMap, path, true)
+}
+
+func alphabeticalOrder[T any](mapping map[string]T) *yamlsaver.Order {
+	var order []string
+	for k := range mapping {
+		order = append(order, k)
+	}
+	slices.Sort(order)
+	return yamlsaver.NewOrder(order)
+}

bundle/internal/annotation/file_test.go

@@ -0,0 +1,123 @@
+package annotation
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoad(t *testing.T) {
+	t.Run("loads valid file", func(t *testing.T) {
+		dir := t.TempDir()
+		path := filepath.Join(dir, "annotations.yml")
+		content := `github.com/databricks/cli/bundle/config.Bundle:
+  name:
+    description: The bundle name
+    since_version: v0.228.0
+  cluster_id:
+    description: The cluster ID
+`
+		err := os.WriteFile(path, []byte(content), 0o644)
+		require.NoError(t, err)
+
+		file, err := Load(path)
+		require.NoError(t, err)
+
+		assert.Equal(t, "The bundle name", file["github.com/databricks/cli/bundle/config.Bundle"]["name"].Description)
+		assert.Equal(t, "v0.228.0", file["github.com/databricks/cli/bundle/config.Bundle"]["name"].SinceVersion)
+		assert.Equal(t, "The cluster ID", file["github.com/databricks/cli/bundle/config.Bundle"]["cluster_id"].Description)
+	})
+
+	t.Run("returns empty file for nonexistent path", func(t *testing.T) {
+		file, err := Load("/nonexistent/path/annotations.yml")
+		require.NoError(t, err)
+		assert.Empty(t, file)
+	})
+
+	t.Run("returns empty file for empty content", func(t *testing.T) {
+		dir := t.TempDir()
+		path := filepath.Join(dir, "empty.yml")
+		err := os.WriteFile(path, []byte(""), 0o644)
+		require.NoError(t, err)
+
+		file, err := Load(path)
+		require.NoError(t, err)
+		assert.Empty(t, file)
+	})
+
+	t.Run("returns error for invalid yaml", func(t *testing.T) {
+		dir := t.TempDir()
+		path := filepath.Join(dir, "invalid.yml")
+		err := os.WriteFile(path, []byte("not: valid: yaml: content"), 0o644)
+		require.NoError(t, err)
+
+		_, err = Load(path)
+		assert.Error(t, err)
+	})
+}
+
+func TestSave(t *testing.T) {
+	t.Run("saves file with annotations", func(t *testing.T) {
+		dir := t.TempDir()
+		path := filepath.Join(dir, "annotations.yml")
+
+		file := File{
+			"github.com/databricks/cli/bundle/config.Bundle": {
+				"name": Descriptor{
+					Description:  "The bundle name",
+					SinceVersion: "v0.228.0",
+				},
+				"cluster_id": Descriptor{
+					Description: "The cluster ID",
+				},
+			},
+		}
+
+		err := file.Save(path)
+		require.NoError(t, err)
+
+		// Verify by loading it back
+		loaded, err := Load(path)
+		require.NoError(t, err)
+
+		assert.Equal(t, "The bundle name", loaded["github.com/databricks/cli/bundle/config.Bundle"]["name"].Description)
+		assert.Equal(t, "v0.228.0", loaded["github.com/databricks/cli/bundle/config.Bundle"]["name"].SinceVersion)
+		assert.Equal(t, "The cluster ID", loaded["github.com/databricks/cli/bundle/config.Bundle"]["cluster_id"].Description)
+	})
+
+	t.Run("sorts types alphabetically", func(t *testing.T) {
+		dir := t.TempDir()
+		path := filepath.Join(dir, "annotations.yml")
+
+		file := File{
+			"z_type": {"field": Descriptor{Description: "z"}},
+			"a_type": {"field": Descriptor{Description: "a"}},
+			"m_type": {"field": Descriptor{Description: "m"}},
+		}
+
+		err := file.Save(path)
+		require.NoError(t, err)
+
+		content, err := os.ReadFile(path)
+		require.NoError(t, err)
+
+		aIdx := indexOf(string(content), "a_type:")
+		mIdx := indexOf(string(content), "m_type:")
+		zIdx := indexOf(string(content), "z_type:")
+
+		assert.Less(t, aIdx, mIdx, "a_type should come before m_type")
+		assert.Less(t, mIdx, zIdx, "m_type should come before z_type")
+	})
+}
+
+func indexOf(s, substr string) int {
+	for i := 0; i <= len(s)-len(substr); i++ {
+		if s[i:i+len(substr)] == substr {
+			return i
+		}
+	}
+	return -1
+}

bundle/internal/schema/.last_processed_cli_version

@@ -0,0 +1 @@
+v0.281.0

bundle/internal/schema/README.md

@@ -0,0 +1,26 @@
+# Bundle Schema Generator
+
+This package generates the JSON schema for Databricks Asset Bundles configuration.
+
+## Annotation Files
+
+The schema generator uses three YAML files to add descriptions and metadata to the generated schema:
+
+- **annotations_openapi.yml**: Auto-generated from the Databricks OpenAPI spec. Contains descriptions for SDK types (jobs, pipelines, etc.). Do not edit manually.
+
+- **annotations_openapi_overrides.yml**: Manual overrides for OpenAPI annotations. Use this to fix or enhance descriptions from the OpenAPI spec without modifying the auto-generated file.
+
+- **annotations.yml**: Manual annotations for CLI-specific types (e.g., `bundle`, `workspace`, `artifacts`). Missing annotations are automatically added with `PLACEHOLDER` descriptions.
+
+## Annotation Priority
+
+Files are merged in order, with later files taking precedence:
+1. `annotations_openapi.yml` (base)
+2. `annotations_openapi_overrides.yml` (overrides OpenAPI)
+3. `annotations.yml` (CLI-specific, highest priority)
+
+## Usage
+
+Run `make schema` from the repository root to regenerate the bundle JSON schema.
+
+To update OpenAPI annotations, set `DATABRICKS_OPENAPI_SPEC` to the path of the OpenAPI spec file before running.

bundle/internal/schema/annotations.go

@@ -6,17 +6,13 @@ import (
 	"os"
 	"reflect"
 	"regexp"
-	"slices"
 	"strings"
 
-	yaml3 "gopkg.in/yaml.v3"
-
 	"github.com/databricks/cli/bundle/internal/annotation"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/dyn/convert"
 	"github.com/databricks/cli/libs/dyn/merge"
 	"github.com/databricks/cli/libs/dyn/yamlloader"
-	"github.com/databricks/cli/libs/dyn/yamlsaver"
 	"github.com/databricks/cli/libs/jsonschema"
 )
 
@@ -108,11 +104,7 @@ func (d *annotationHandler) syncWithMissingAnnotations(outputPath string) error
 		return err
 	}
 
-	err = saveYamlWithStyle(outputPath, outputTyped)
-	if err != nil {
-		return err
-	}
-	return nil
+	return outputTyped.Save(outputPath)
 }
 
 func getPath(typ reflect.Type) string {
@@ -145,50 +137,10 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
 	s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription)
 	s.Title = a.Title
 	s.Enum = a.Enum
-}
-
-func saveYamlWithStyle(outputPath string, annotations annotation.File) error {
-	annotationOrder := yamlsaver.NewOrder([]string{"description", "markdown_description", "title", "default", "enum"})
-	style := map[string]yaml3.Style{}
-
-	order := getAlphabeticalOrder(annotations)
-	dynMap := map[string]dyn.Value{}
-	for k, v := range annotations {
-		style[k] = yaml3.LiteralStyle
-
-		properties := map[string]dyn.Value{}
-		propertiesOrder := getAlphabeticalOrder(v)
-		for key, value := range v {
-			d, err := convert.FromTyped(value, dyn.NilValue)
-			if d.Kind() == dyn.KindNil || err != nil {
-				properties[key] = dyn.NewValue(map[string]dyn.Value{}, []dyn.Location{{Line: propertiesOrder.Get(key)}})
-				continue
-			}
-			val, err := yamlsaver.ConvertToMapValue(value, annotationOrder, []string{}, map[string]dyn.Value{})
-			if err != nil {
-				return err
-			}
-			properties[key] = val.WithLocations([]dyn.Location{{Line: propertiesOrder.Get(key)}})
-		}
-
-		dynMap[k] = dyn.NewValue(properties, []dyn.Location{{Line: order.Get(k)}})
-	}
-
-	saver := yamlsaver.NewSaverWithStyle(style)
-	err := saver.SaveAsYAML(dynMap, outputPath, true)
-	if err != nil {
-		return err
-	}
-	return nil
-}
 
-func getAlphabeticalOrder[T any](mapping map[string]T) *yamlsaver.Order {
-	var order []string
-	for k := range mapping {
-		order = append(order, k)
+	if a.SinceVersion != "" {
+		s.SinceVersion = a.SinceVersion
 	}
-	slices.Sort(order)
-	return yamlsaver.NewOrder(order)
 }
 
 func convertLinksToAbsoluteUrl(s string) string {