Add entity type schemas

api/api_gomux.go

@@ -28,6 +28,7 @@ import (
 	"github.com/square/etre/entity"
 	"github.com/square/etre/metrics"
 	"github.com/square/etre/query"
+	"github.com/square/etre/schema"
 )
 
 func init() {
@@ -65,6 +66,7 @@ type API struct {
 	queryProfSampleRate      int
 	queryProfReportThreshold time.Duration
 	srv                      *http.Server
+	schemas                  schema.Config
 }
 
 // NewAPI godoc
@@ -94,6 +96,7 @@ func NewAPI(appCtx app.Context) *API {
 		queryLatencySLA:          queryLatencySLA,
 		queryProfSampleRate:      int(appCtx.Config.Metrics.QueryProfileSampleRate * 100),
 		queryProfReportThreshold: queryProfReportThreshold,
+		schemas:                  appCtx.Config.Schemas,
 	}
 
 	mux := http.NewServeMux()
@@ -120,6 +123,17 @@ func NewAPI(appCtx app.Context) *API {
 	mux.Handle("GET "+etre.API_ROOT+"/entity/{type}/{id}/labels", api.requestWrapper(api.id(http.HandlerFunc(api.getLabelsHandler))))
 	mux.Handle("DELETE "+etre.API_ROOT+"/entity/{type}/{id}/labels/{label}", api.requestWrapper(api.id(http.HandlerFunc(api.deleteLabelHandler))))
 
+	// /////////////////////////////////////////////////////////////////////
+	// Schemas
+	// /////////////////////////////////////////////////////////////////////
+	mux.HandleFunc("GET "+etre.API_ROOT+"/schemas/{type}", api.getSchemasHandler)
+	mux.HandleFunc("GET "+etre.API_ROOT+"/schemas", api.getSchemasHandler)
+
+	// /////////////////////////////////////////////////////////////////////
+	// Entity Types
+	// /////////////////////////////////////////////////////////////////////
+	mux.HandleFunc("GET "+etre.API_ROOT+"/entity-types", api.getEntityTypesHandler)
+
 	// /////////////////////////////////////////////////////////////////////
 	// Metrics and status
 	// /////////////////////////////////////////////////////////////////////
@@ -1173,6 +1187,52 @@ func (api *API) changesHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// getSchemasHandler godoc
+// @Summary Get entity schemas
+// @Description Return the schema for one entity of the given :type
+// @ID getEntityHandler
+// @Produce json
+// @Param type path string true "Entity type"
+// @Success 200 {object} schema.Config "OK"
+// @Failure 400,404 {object} etre.Error
+// @Router /schema/:type [get]
+func (api *API) getSchemasHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	// Get the schemas config for all schemas
+	cfg := api.schemas
+
+	// If entity type is provided, make sure it is valid
+	entityType := r.PathValue("type")
+	if entityType != "" {
+		if err := api.validate.EntityType(entityType); err != nil {
+			log.Printf("Invalid entity type: '%s': request=%+v", entityType, r)
+			w.WriteHeader(http.StatusBadRequest)
+			json.NewEncoder(w).Encode(err) // validation error will encode nicely
+			return
+		}
+		// Make a config with only the requested entity type schema.
+		cfg.Entities = map[string]schema.EntitySchema{
+			entityType: api.schemas.Entities[entityType],
+		}
+	}
+
+	// Return the schema(s)
+	json.NewEncoder(w).Encode(cfg)
+}
+
+// getEntityTypesHandler godoc
+// @Summary Get supported entity types
+// @Description Return a list of all supported entity types
+// @ID getEntityTypesHandler
+// @Produce json
+// @Success 200 {array} string "List of entity types"
+// @Router /entity-types [get]
+func (api *API) getEntityTypesHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	entityTypes := api.validate.EntityTypes()
+	json.NewEncoder(w).Encode(entityTypes)
+}
+
 // Return error on read. Writes always return an etre.WriteResult by calling WriteResult.
 func (api *API) readError(rc *req, w http.ResponseWriter, err error) {
 	api.systemMetrics.Inc(metrics.Error, 1)

api/entity_types_test.go

@@ -0,0 +1,104 @@
+package api_test
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/square/etre"
+	"github.com/square/etre/api"
+	"github.com/square/etre/app"
+	"github.com/square/etre/auth"
+	"github.com/square/etre/config"
+	"github.com/square/etre/entity"
+	"github.com/square/etre/metrics"
+	srv "github.com/square/etre/server"
+	"github.com/square/etre/test"
+	"github.com/square/etre/test/mock"
+)
+
+func TestGetEntityTypes(t *testing.T) {
+	tests := []struct {
+		name        string
+		entityTypes []string
+		expectTypes []string
+	}{
+		{
+			name:        "Single entity type",
+			entityTypes: []string{"nodes"},
+			expectTypes: []string{"nodes"},
+		},
+		{
+			name:        "Multiple entity types",
+			entityTypes: []string{"nodes", "racks", "hosts"},
+			expectTypes: []string{"nodes", "racks", "hosts"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Set up the server with custom entity types
+			config := defaultConfig
+			server := setupWithValidator(t, config, mock.EntityStore{}, entity.NewValidator(tt.entityTypes))
+			defer server.ts.Close()
+
+			// Set up the request URL
+			etreurl := server.url + etre.API_ROOT + "/entity-types"
+
+			// Make the HTTP call
+			var gotTypes []string
+			statusCode, err := test.MakeHTTPRequest("GET", etreurl, nil, &gotTypes)
+			require.NoError(t, err)
+			assert.Equal(t, http.StatusOK, statusCode, "response status = %d, expected %d, url %s", statusCode, http.StatusOK, etreurl)
+
+			// Make sure we got the expected entity types
+			assert.Equal(t, tt.expectTypes, gotTypes)
+		})
+	}
+}
+
+func setupWithValidator(t *testing.T, cfg config.Config, store mock.EntityStore, validator entity.Validator) *server {
+	etre.DebugEnabled = true
+
+	server := &server{
+		store:           store,
+		cfg:             cfg,
+		auth:            &mock.AuthRecorder{},
+		cdcStore:        &mock.CDCStore{},
+		streamerFactory: &mock.StreamerFactory{},
+		metricsrec:      mock.NewMetricsRecorder(),
+		sysmetrics:      mock.NewMetricsRecorder(),
+	}
+
+	acls, err := srv.MapConfigACLRoles(cfg.Security.ACL)
+	require.NoError(t, err, "invalid Config.ACL: %s", err)
+
+	ms := metrics.NewMemoryStore()
+	mf := metrics.GroupFactory{Store: ms}
+	sm := metrics.NewSystemMetrics()
+
+	appCtx := app.Context{
+		Config:          server.cfg,
+		EntityStore:     server.store,
+		EntityValidator: validator,
+		Auth:            auth.NewManager(acls, server.auth),
+		MetricsStore:    ms,
+		MetricsFactory:  mock.NewMetricsFactory(mf, server.metricsrec),
+		StreamerFactory: server.streamerFactory,
+		SystemMetrics:   mock.NewSystemMetrics(sm, server.sysmetrics),
+	}
+	server.api = api.NewAPI(appCtx)
+	server.ts = httptest.NewServer(server.api)
+
+	u, err := url.Parse(server.ts.URL)
+	require.NoError(t, err)
+
+	server.url = fmt.Sprintf("http://%s", u.Host)
+
+	return server
+}

api/get_schemas_test.go

@@ -0,0 +1,134 @@
+package api_test
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/square/etre"
+	"github.com/square/etre/schema"
+	"github.com/square/etre/test"
+	"github.com/square/etre/test/mock"
+)
+
+func TestGetSchemas(t *testing.T) {
+	tests := []struct {
+		name       string
+		entityType string
+		config     schema.Config
+		expect     schema.Config
+	}{
+		{
+			name:       "No schemas configured",
+			entityType: "nodes",
+			expect:     schema.Config{Entities: map[string]schema.EntitySchema{"nodes": {}}}, // the server will return an empty schema for known types
+		},
+		{
+			name: "No schemas configured - No type param",
+		},
+		{
+			name:       "Schema configured - type param present",
+			entityType: "nodes",
+			config: schema.Config{
+				Entities: map[string]schema.EntitySchema{
+					"nodes": {
+						Schema: &schema.Schema{
+							Fields: []schema.Field{
+								{Name: "hostname", Type: "string", Required: true},
+								{Name: "status", Type: "string", Required: false},
+							},
+							AdditionalProperties: true,
+							ValidationLevel:      "strict",
+						},
+					},
+				},
+			},
+			expect: schema.Config{
+				Entities: map[string]schema.EntitySchema{
+					"nodes": {
+						Schema: &schema.Schema{
+							Fields: []schema.Field{
+								{Name: "hostname", Type: "string", Required: true},
+								{Name: "status", Type: "string", Required: false},
+							},
+							AdditionalProperties: true,
+							ValidationLevel:      "strict",
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "Schema configured - no type param",
+			config: schema.Config{
+				Entities: map[string]schema.EntitySchema{
+					"nodes": {
+						Schema: &schema.Schema{
+							Fields: []schema.Field{
+								{Name: "hostname", Type: "string", Required: true},
+							},
+							AdditionalProperties: false,
+						},
+					},
+					"racks": {
+						Schema: &schema.Schema{
+							Fields: []schema.Field{
+								{Name: "rack_id", Type: "string", Required: true},
+								{Name: "datacenter", Type: "string", Required: true},
+							},
+							ValidationLevel: "moderate",
+						},
+					},
+				},
+			},
+			expect: schema.Config{
+				Entities: map[string]schema.EntitySchema{
+					"nodes": {
+						Schema: &schema.Schema{
+							Fields: []schema.Field{
+								{Name: "hostname", Type: "string", Required: true},
+							},
+							AdditionalProperties: false,
+						},
+					},
+					"racks": {
+						Schema: &schema.Schema{
+							Fields: []schema.Field{
+								{Name: "rack_id", Type: "string", Required: true},
+								{Name: "datacenter", Type: "string", Required: true},
+							},
+							ValidationLevel: "moderate",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Set up the server
+			config := defaultConfig
+			config.Schemas = tt.config
+			server := setup(t, config, mock.EntityStore{})
+			defer server.ts.Close()
+
+			// Set up the request URL
+			etreurl := server.url + etre.API_ROOT + "/schemas"
+			if tt.entityType != "" {
+				etreurl += "/" + tt.entityType
+			}
+
+			// Make the HTTP call
+			var gotSchemas schema.Config
+			statusCode, err := test.MakeHTTPRequest("GET", etreurl, nil, &gotSchemas)
+			require.NoError(t, err)
+			assert.Equal(t, http.StatusOK, statusCode, "response status = %d, expected %d, url %s", statusCode, http.StatusOK, etreurl)
+
+			// Make sure we got the expected schemas
+			assert.Equal(t, tt.expect, gotSchemas)
+		})
+	}
+}

config/config.go

@@ -9,6 +9,8 @@ import (
 	"strings"
 
 	"gopkg.in/yaml.v2"
+
+	"github.com/square/etre/schema"
 )
 
 const (
@@ -120,6 +122,7 @@ type Config struct {
 	CDC        CDCConfig        `yaml:"cdc"`
 	Security   SecurityConfig   `yaml:"security"`
 	Metrics    MetricsConfig    `yaml:"metrics"`
+	Schemas    schema.Config    `yaml:"schemas"`
 }
 
 func Redact(c Config) Config {

entity/validate.go

@@ -29,6 +29,7 @@ type Validator interface {
 	Entities([]etre.Entity, byte) error
 	WriteOp(WriteOp) error
 	DeleteLabel(string) error
+	EntityTypes() []string
 }
 
 type validator struct {
@@ -152,3 +153,7 @@ func (v validator) DeleteLabel(label string) error {
 	}
 	return nil
 }
+
+func (v validator) EntityTypes() []string {
+	return v.entityTypes
+}

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/go-test/deep v1.1.1
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8
 	github.com/gorilla/websocket v1.5.3
+	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.9.0
 	github.com/swaggo/http-swagger v1.3.4
 	github.com/swaggo/swag v1.16.4

go.sum

@@ -36,6 +36,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=