Refactor recorder cache

server/controller/common/resource_type.go

@@ -70,6 +70,7 @@ const (
 	RESOURCE_TYPE_IP_EN      = "ip"
 	RESOURCE_TYPE_ALL_IP_EN  = "all_ip"
 	RESOURCE_TYPE_LB_RULE_EN = "lb_rule"
+	RESOURCE_TYPE_AGENT_EN   = "agent"
 
 	RESOURCE_TYPE_CUSTOM_SERVICE_EN = "biz_service"
 

server/controller/recorder/cache/diffbase/az.go

@@ -17,39 +17,35 @@
 package diffbase
 
 import (
-	cloudmodel "github.com/deepflowio/deepflow/server/controller/cloud/model"
 	ctrlrcommon "github.com/deepflowio/deepflow/server/controller/common"
 	metadbmodel "github.com/deepflowio/deepflow/server/controller/db/metadb/model"
+	"github.com/deepflowio/deepflow/server/controller/recorder/cache/tool"
 )
 
-func (b *DataSet) AddAZ(dbItem *metadbmodel.AZ, seq int) {
-	b.AZs[dbItem.Lcuuid] = &AZ{
-		DiffBase: DiffBase{
-			Sequence: seq,
-			Lcuuid:   dbItem.Lcuuid,
-		},
-		Name:         dbItem.Name,
-		Label:        dbItem.Label,
-		RegionLcuuid: dbItem.Region,
-	}
-	b.GetLogFunc()(addDiffBase(ctrlrcommon.RESOURCE_TYPE_AZ_EN, b.AZs[dbItem.Lcuuid]), b.metadata.LogPrefixes)
+type AZ struct {
+	ResourceBase
+	Name         string
+	Label        string
+	RegionLcuuid string
 }
 
-func (b *DataSet) DeleteAZ(lcuuid string) {
-	delete(b.AZs, lcuuid)
-	log.Info(deleteDiffBase(ctrlrcommon.RESOURCE_TYPE_AZ_EN, lcuuid), b.metadata.LogPrefixes)
+func (a *AZ) reset(dbItem *metadbmodel.AZ, tool *tool.Tool) {
+	a.Name = dbItem.Name
+	a.Label = dbItem.Label
+	a.RegionLcuuid = dbItem.Region
 }
 
-type AZ struct {
-	DiffBase
-	Name         string `json:"name"`
-	Label        string `json:"label"`
-	RegionLcuuid string `json:"region_lcuuid"`
+func NewAZCollection(t *tool.Tool) *AZCollection {
+	c := new(AZCollection)
+	c.collection = newCollectionBuilder[*AZ]().
+		withResourceType(ctrlrcommon.RESOURCE_TYPE_AZ_EN).
+		withTool(t).
+		withDBItemFactory(func() *metadbmodel.AZ { return new(metadbmodel.AZ) }).
+		withCacheItemFactory(func() *AZ { return new(AZ) }).
+		build()
+	return c
 }
 
-func (a *AZ) Update(cloudItem *cloudmodel.AZ) {
-	a.Name = cloudItem.Name
-	a.Label = cloudItem.Label
-	a.RegionLcuuid = cloudItem.RegionLcuuid
-	log.Info(updateDiffBase(ctrlrcommon.RESOURCE_TYPE_AZ_EN, a))
+type AZCollection struct {
+	collection[*AZ, *metadbmodel.AZ]
 }

server/controller/recorder/cache/diffbase/cen.go

@@ -24,16 +24,16 @@ import (
 	rcommon "github.com/deepflowio/deepflow/server/controller/recorder/common"
 )
 
-func (b *DataSet) AddCEN(dbItem *metadbmodel.CEN, seq int, toolDataSet *tool.DataSet) {
+func (b *DataSet) AddCEN(dbItem *metadbmodel.CEN, seq int, tool *tool.Tool) {
 	vpcLcuuids := []string{}
 	for _, vpcID := range rcommon.StringToIntSlice(dbItem.VPCIDs) {
-		vpcLcuuid, exists := toolDataSet.GetVPCLcuuidByID(vpcID)
-		if exists {
+		vpcLcuuid := tool.VPC().GetByID(vpcID).Lcuuid()
+		if vpcLcuuid != "" {
 			vpcLcuuids = append(vpcLcuuids, vpcLcuuid)
 		}
 	}
 	b.CENs[dbItem.Lcuuid] = &CEN{
-		DiffBase: DiffBase{
+		ResourceBase: ResourceBase{
 			Sequence: seq,
 			Lcuuid:   dbItem.Lcuuid,
 		},
@@ -48,14 +48,38 @@ func (b *DataSet) DeleteCEN(lcuuid string) {
 	log.Info(deleteDiffBase(ctrlrcommon.RESOURCE_TYPE_CEN_EN, lcuuid), b.metadata.LogPrefixes)
 }
 
+func (c *CEN) Update(cloudItem *cloudmodel.CEN) {
+	c.Name = cloudItem.Name
+	c.VPCLcuuids = cloudItem.VPCLcuuids
+	log.Info(updateDiffBase(ctrlrcommon.RESOURCE_TYPE_CEN_EN, c))
+}
+
 type CEN struct {
-	DiffBase
+	ResourceBase
 	Name       string   `json:"name"`
 	VPCLcuuids []string `json:"vpc_lcuuids"`
 }
 
-func (c *CEN) Update(cloudItem *cloudmodel.CEN) {
-	c.Name = cloudItem.Name
-	c.VPCLcuuids = cloudItem.VPCLcuuids
-	log.Info(updateDiffBase(ctrlrcommon.RESOURCE_TYPE_CEN_EN, c))
+func (c *CEN) reset(dbItem *metadbmodel.CEN, tool *tool.Tool) {
+	vpcLcuuids := []string{}
+	for _, vpcID := range rcommon.StringToIntSlice(dbItem.VPCIDs) {
+		vpcLcuuid := tool.VPC().GetByID(vpcID).Lcuuid()
+		if vpcLcuuid != "" {
+			vpcLcuuids = append(vpcLcuuids, vpcLcuuid)
+		}
+	}
+	c.Name = dbItem.Name
+	c.VPCLcuuids = vpcLcuuids
+}
+
+type CENCollection struct {
+	collectionComponent[*CEN, CEN, *metadbmodel.CEN, metadbmodel.CEN]
+}
+
+func NewCENCollection(t *tool.Tool) *CENCollection {
+	c := new(CENCollection)
+	c.withResourceType(ctrlrcommon.RESOURCE_TYPE_CEN_EN).
+		withTool(t).
+		init()
+	return c
 }

server/controller/recorder/cache/diffbase/collection.go

@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2024 Yunshan Networks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package diffbase
+
+import (
+	"github.com/deepflowio/deepflow/server/controller/recorder/cache/tool"
+)
+
+type ResourceBase struct {
+	Sequence int    `json:"sequence"`
+	Lcuuid   string `json:"lcuuid"`
+}
+
+func (d ResourceBase) GetSequence() int {
+	return d.Sequence
+}
+
+func (d ResourceBase) GetLcuuid() string {
+	return d.Lcuuid
+}
+
+func (d *ResourceBase) SetSequence(sequence int) {
+	d.Sequence = sequence
+}
+
+func (d *ResourceBase) init(sequence int, lcuuid string) {
+	d.Sequence = sequence
+	d.Lcuuid = lcuuid
+}
+
+// CacheItem defines cache object must implement the interface
+type CacheItem[D DBItem] interface {
+	GetLcuuid() string
+	GetSequence() int
+	init(sequence int, lcuuid string)
+	reset(dbItem D, tool *tool.Tool) // reset the item with dbItem
+}
+
+// DBItem defines database object must implement the interface
+type DBItem interface {
+	GetID() int
+	GetLcuuid() string
+}
+
+// collectionBuilder use builder pattern to build collection
+type collectionBuilder[T CacheItem[D], D DBItem] struct {
+	collection[T, D]
+}
+
+func newCollectionBuilder[T CacheItem[D], D DBItem]() *collectionBuilder[T, D] {
+	return &collectionBuilder[T, D]{}
+}
+
+func (b *collectionBuilder[T, D]) withResourceType(resourceType string) *collectionBuilder[T, D] {
+	b.resourceType = resourceType
+	return b
+}
+
+func (b *collectionBuilder[T, D]) withTool(tool *tool.Tool) *collectionBuilder[T, D] {
+	b.tool = tool
+	return b
+}
+
+func (b *collectionBuilder[T, D]) withDBItemFactory(factory func() D) *collectionBuilder[T, D] {
+	b.dbItemFactory = factory
+	return b
+}
+
+func (b *collectionBuilder[T, D]) withCacheItemFactory(factory func() T) *collectionBuilder[T, D] {
+	b.cacheItemFactory = factory
+	return b
+}
+
+func (b *collectionBuilder[T, D]) build() collection[T, D] {
+	return collection[T, D]{
+		resourceType:     b.resourceType,
+		tool:             b.tool,
+		dbItemFactory:    b.dbItemFactory,
+		cacheItemFactory: b.cacheItemFactory,
+
+		lcuuidToItem: make(map[string]T),
+	}
+}
+
+type collection[T CacheItem[D], D DBItem] struct {
+	resourceType string
+	tool         *tool.Tool
+
+	dbItemFactory    func() D
+	cacheItemFactory func() T
+
+	lcuuidToItem map[string]T
+}
+
+func (c *collection[T, D]) GetByLcuuid(lcuuid string) T {
+	empty := c.cacheItemFactory()
+	if lcuuid == "" {
+		return empty
+	}
+	if item, ok := c.lcuuidToItem[lcuuid]; ok {
+		return item
+	}
+	return empty
+}
+
+func (c *collection[T, D]) GetAll() map[string]T {
+	return c.lcuuidToItem
+}
+
+func (c *collection[T, D]) Add(dbItem D, seq int) {
+	item := c.cacheItemFactory()
+	item.init(seq, dbItem.GetLcuuid())
+	item.reset(dbItem, c.tool)
+	c.lcuuidToItem[dbItem.GetLcuuid()] = item
+	c.tool.GetLogFunc()(addDiffBase(c.resourceType, dbItem.GetLcuuid()), c.tool.Metadata().LogPrefixes)
+}
+
+func (c *collection[T, D]) Update(dbItem D, seq int) {
+	if existingItem, ok := c.lcuuidToItem[dbItem.GetLcuuid()]; ok {
+		existingItem.reset(dbItem, c.tool)
+		c.tool.GetLogFunc()(updateDiffBase(c.resourceType, dbItem.GetLcuuid()), c.tool.Metadata().LogPrefixes)
+		return
+	}
+	// if cache item not exists, add it
+	log.Errorf("%s cache item not found (lcuuid: %s), add it", c.resourceType, dbItem.GetLcuuid(), c.tool.Metadata().LogPrefixes)
+	c.Add(dbItem, seq)
+}
+
+func (c *collection[T, D]) Delete(dbItem D) {
+	delete(c.lcuuidToItem, dbItem.GetLcuuid())
+	c.tool.GetLogFunc()(deleteDiffBase(c.resourceType, dbItem.GetLcuuid()), c.tool.Metadata().LogPrefixes)
+}

server/controller/recorder/cache/diffbase/config_map.go

@@ -17,54 +17,42 @@
 package diffbase
 
 import (
-	"sigs.k8s.io/yaml"
-
-	cloudmodel "github.com/deepflowio/deepflow/server/controller/cloud/model"
 	ctrlrcommon "github.com/deepflowio/deepflow/server/controller/common"
 	metadbmodel "github.com/deepflowio/deepflow/server/controller/db/metadb/model"
 	"github.com/deepflowio/deepflow/server/controller/recorder/cache/tool"
 )
 
-func (b *DataSet) AddConfigMap(dbItem *metadbmodel.ConfigMap, seq int) {
-	b.ConfigMaps[dbItem.Lcuuid] = &ConfigMap{
-		DiffBase: DiffBase{
-			Sequence: seq,
-			Lcuuid:   dbItem.Lcuuid,
-		},
-		Name:     dbItem.Name,
-		Data:     string(dbItem.Data),
-		DataHash: dbItem.DataHash,
-	}
-	b.GetLogFunc()(addDiffBase(ctrlrcommon.RESOURCE_TYPE_CONFIG_MAP_EN, b.ConfigMaps[dbItem.Lcuuid].ToLoggable()), b.metadata.LogPrefixes)
+type ConfigMap struct {
+	ResourceBase
+	Name     string
+	Data     string
+	DataHash string
 }
 
-func (b *DataSet) DeleteConfigMap(lcuuid string) {
-	delete(b.ConfigMaps, lcuuid)
-	log.Info(deleteDiffBase(ctrlrcommon.RESOURCE_TYPE_CONFIG_MAP_EN, lcuuid), b.metadata.LogPrefixes)
+func (a *ConfigMap) reset(dbItem *metadbmodel.ConfigMap, tool *tool.Tool) {
+	a.Name = dbItem.Name
+	a.Data = string(dbItem.Data)
+	a.DataHash = dbItem.DataHash
 }
 
-type ConfigMap struct {
-	DiffBase
-	Name     string `json:"name"`
-	Data     string `json:"data"`
-	DataHash string `json:"data_hash"`
-}
-
-// ToLoggable converts the ConfigMap to a loggable format, excluding the Data field.
-func (v ConfigMap) ToLoggable() interface{} {
-	copied := v
+// ToLoggable converts ConfigMap to a loggable format, excluding sensitive fields
+func (a ConfigMap) ToLoggable() interface{} {
+	copied := a
 	copied.Data = "**HIDDEN**"
 	return copied
 }
 
-func (v *ConfigMap) Update(cloudItem *cloudmodel.ConfigMap, toolDataSet *tool.DataSet) {
-	v.Name = cloudItem.Name
-	yamlData, err := yaml.JSONToYAML([]byte(cloudItem.Data))
-	if err != nil {
-		log.Errorf("failed to convert JSON data: %v to YAML: %s", cloudItem.Data, toolDataSet.GetMetadata().LogPrefixes)
-		return
-	}
-	v.Data = string(yamlData)
-	v.DataHash = cloudItem.DataHash
-	log.Info(updateDiffBase(ctrlrcommon.RESOURCE_TYPE_CONFIG_MAP_EN, v.ToLoggable()))
+func NewConfigMapCollection(t *tool.Tool) *ConfigMapCollection {
+	c := new(ConfigMapCollection)
+	c.collection = newCollectionBuilder[*ConfigMap]().
+		withResourceType(ctrlrcommon.RESOURCE_TYPE_CONFIG_MAP_EN).
+		withTool(t).
+		withDBItemFactory(func() *metadbmodel.ConfigMap { return new(metadbmodel.ConfigMap) }).
+		withCacheItemFactory(func() *ConfigMap { return new(ConfigMap) }).
+		build()
+	return c
+}
+
+type ConfigMapCollection struct {
+	collection[*ConfigMap, *metadbmodel.ConfigMap]
 }