[Draft] Support version reference

api/pkg/apis/v1alpha1/managers/configs/configs-manager.go

@@ -58,8 +58,9 @@ func (s *ConfigsManager) Init(context *contexts.VendorContext, cfg managers.Mana
 	return nil
 }
 func (s *ConfigsManager) Get(object string, field string, overlays []string, localContext interface{}) (interface{}, error) {
-	if strings.Index(object, ":") > 0 {
-		parts := strings.Split(object, ":")
+	log.Debugf(" M (Config): Get %v, config provider size %d", object, len(s.ConfigProviders))
+	if strings.Index(object, "::") > 0 {
+		parts := strings.Split(object, "::")
 		if len(parts) != 2 {
 			return "", v1alpha2.NewCOAError(nil, fmt.Sprintf("Invalid object: %s", object), v1alpha2.BadRequest)
 		}
@@ -122,8 +123,9 @@ func (s *ConfigsManager) getWithOverlay(provider config.IConfigProvider, object
 }
 
 func (s *ConfigsManager) GetObject(object string, overlays []string, localContext interface{}) (map[string]interface{}, error) {
-	if strings.Index(object, ":") > 0 {
-		parts := strings.Split(object, ":")
+	log.Debugf(" M (Config): GetObject %v, config provider size %d", object, len(s.ConfigProviders))
+	if strings.Index(object, "::") > 0 {
+		parts := strings.Split(object, "::")
 		if len(parts) != 2 {
 			return nil, v1alpha2.NewCOAError(nil, fmt.Sprintf("Invalid object: %s", object), v1alpha2.BadRequest)
 		}
@@ -161,8 +163,9 @@ func (s *ConfigsManager) getObjectWithOverlay(provider config.IConfigProvider, o
 	return provider.ReadObject(object, localContext)
 }
 func (s *ConfigsManager) Set(object string, field string, value interface{}) error {
-	if strings.Index(object, ":") > 0 {
-		parts := strings.Split(object, ":")
+	log.Debugf(" M (Config): Set %v, config provider size %d", object, len(s.ConfigProviders))
+	if strings.Index(object, "::") > 0 {
+		parts := strings.Split(object, "::")
 		if len(parts) != 2 {
 			return v1alpha2.NewCOAError(nil, fmt.Sprintf("Invalid object: %s", object), v1alpha2.BadRequest)
 		}
@@ -186,8 +189,9 @@ func (s *ConfigsManager) Set(object string, field string, value interface{}) err
 	return v1alpha2.NewCOAError(nil, fmt.Sprintf("Invalid config object or key: %s, %s", object, field), v1alpha2.BadRequest)
 }
 func (s *ConfigsManager) SetObject(object string, values map[string]interface{}) error {
-	if strings.Index(object, ":") > 0 {
-		parts := strings.Split(object, ":")
+	log.Debugf(" M (Config): SetObject %v, config provider size %d", object, len(s.ConfigProviders))
+	if strings.Index(object, "::") > 0 {
+		parts := strings.Split(object, "::")
 		if len(parts) != 2 {
 			return v1alpha2.NewCOAError(nil, fmt.Sprintf("Invalid object: %s", object), v1alpha2.BadRequest)
 		}
@@ -211,8 +215,9 @@ func (s *ConfigsManager) SetObject(object string, values map[string]interface{})
 	return v1alpha2.NewCOAError(nil, fmt.Sprintf("Invalid config object: %s", object), v1alpha2.BadRequest)
 }
 func (s *ConfigsManager) Delete(object string, field string) error {
-	if strings.Index(object, ":") > 0 {
-		parts := strings.Split(object, ":")
+	log.Debugf(" M (Config): Delete %v, config provider size %d", object, len(s.ConfigProviders))
+	if strings.Index(object, "::") > 0 {
+		parts := strings.Split(object, "::")
 		if len(parts) != 2 {
 			return v1alpha2.NewCOAError(nil, fmt.Sprintf("Invalid object: %s", object), v1alpha2.BadRequest)
 		}
@@ -236,8 +241,9 @@ func (s *ConfigsManager) Delete(object string, field string) error {
 	return v1alpha2.NewCOAError(nil, fmt.Sprintf("Invalid config object or key: %s, %s", object, field), v1alpha2.BadRequest)
 }
 func (s *ConfigsManager) DeleteObject(object string) error {
-	if strings.Index(object, ":") > 0 {
-		parts := strings.Split(object, ":")
+	log.Debugf(" M (Config): DeleteObject %v, config provider size %d", object, len(s.ConfigProviders))
+	if strings.Index(object, "::") > 0 {
+		parts := strings.Split(object, "::")
 		if len(parts) != 2 {
 			return v1alpha2.NewCOAError(nil, fmt.Sprintf("Invalid object: %s", object), v1alpha2.BadRequest)
 		}

api/pkg/apis/v1alpha1/managers/configs/configs-manager_test.go

@@ -336,19 +336,19 @@ func TestObjectDeleteWithProviderSpecified(t *testing.T) {
 		"key2": 42,
 		"key3": true,
 	}
-	manager.SetObject("memory:obj", object)
+	manager.SetObject("memory::obj", object)
 
 	// Delete field
-	err = manager.Delete("memory:obj", "key1")
+	err = manager.Delete("memory::obj", "key1")
 	assert.Nil(t, err)
-	val, err := manager.Get("memory:obj", "key1", nil, nil)
+	val, err := manager.Get("memory::obj", "key1", nil, nil)
 	assert.NotNil(t, err)
 	assert.Empty(t, val)
 
 	// Delete object
-	err2 := manager.DeleteObject("memory:obj")
+	err2 := manager.DeleteObject("memory::obj")
 	assert.Nil(t, err2)
-	val2, err2 := manager.GetObject("memory:obj", nil, nil)
+	val2, err2 := manager.GetObject("memory::obj", nil, nil)
 	assert.NotNil(t, err2)
 	assert.Empty(t, val2)
 }
@@ -421,5 +421,46 @@ func TestObjectDeleteWithMoreProviders(t *testing.T) {
 	val2, err2 := manager.GetObject("obj", nil, nil)
 	assert.NotNil(t, err2)
 	assert.Empty(t, val2)
+}
+
+func TestObjectReference(t *testing.T) {
+	provider1 := memory.MemoryConfigProvider{}
+	err := provider1.Init(memory.MemoryConfigProviderConfig{})
+	assert.Nil(t, err)
+	provider2 := memory.MemoryConfigProvider{}
+	err = provider2.Init(memory.MemoryConfigProviderConfig{})
+	assert.Nil(t, err)
+	manager := ConfigsManager{
+		ConfigProviders: map[string]config.IConfigProvider{
+			"memory1": &provider1,
+			"memory2": &provider2,
+		},
+		Precedence: []string{"memory1", "memory2"},
+	}
+	assert.Nil(t, err)
 
+	object := map[string]interface{}{
+		"key1": "value1",
+		"key2": 42,
+		"key3": true,
+	}
+	// Get field
+	manager.SetObject("memory1::obj:v1", object)
+	val, err := manager.Get("obj:v1", "key1", nil, nil)
+	assert.Nil(t, err)
+	assert.Equal(t, "value1", val)
+
+	// Delete field
+	err = manager.Delete("memory1::obj:v1", "key1")
+	assert.Nil(t, err)
+	val, err = manager.Get("memory1::obj:v1", "key1", nil, nil)
+	assert.NotNil(t, err)
+	assert.Empty(t, val)
+
+	// Delete object
+	err2 := manager.DeleteObject("memory1::obj:v1")
+	assert.Nil(t, err2)
+	val2, err2 := manager.GetObject("memory1::obj:v1", nil, nil)
+	assert.NotNil(t, err2)
+	assert.Empty(t, val2)
 }

api/pkg/apis/v1alpha1/managers/jobs/jobs-manager.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/model"
 	"github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/utils"
+	api_utils "github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/utils"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2/contexts"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2/managers"
@@ -379,7 +380,8 @@ func (s *JobsManager) HandleJobEvent(ctx context.Context, event v1alpha2.Event)
 
 			//get solution
 			var solution model.SolutionState
-			solution, err = s.apiClient.GetSolution(ctx, instance.Spec.Solution, namespace, s.user, s.password)
+			solutionName := api_utils.ReplaceSeperator(instance.Spec.Solution)
+			solution, err = s.apiClient.GetSolution(ctx, solutionName, namespace, s.user, s.password)
 			if err != nil {
 				solution = model.SolutionState{
 					ObjectMeta: model.ObjectMeta{

api/pkg/apis/v1alpha1/providers/config/catalog/catalogprovider.go

@@ -14,13 +14,16 @@ import (
 
 	"github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/model"
 	"github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/utils"
+	api_utils "github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/utils"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2/contexts"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2/providers"
 	coa_utils "github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2/utils"
+	"github.com/eclipse-symphony/symphony/coa/pkg/logger"
 )
 
 var msLock sync.Mutex
+var clog = logger.NewLogger("coa.runtime")
 
 type CatalogConfigProviderConfig struct {
 	User     string `json:"user"`
@@ -87,6 +90,7 @@ func CatalogConfigProviderConfigFromMap(properties map[string]string) (CatalogCo
 	return ret, nil
 }
 func (m *CatalogConfigProvider) unwindOverrides(override string, field string, namespace string) (string, error) {
+	override = api_utils.ReplaceSeperator(override)
 	catalog, err := m.ApiClient.GetCatalog(context.TODO(), override, namespace, m.Config.User, m.Config.Password)
 	if err != nil {
 		return "", err
@@ -104,8 +108,9 @@ func (m *CatalogConfigProvider) unwindOverrides(override string, field string, n
 	return "", v1alpha2.NewCOAError(nil, fmt.Sprintf("field '%s' is not found in configuration '%s'", field, override), v1alpha2.NotFound)
 }
 func (m *CatalogConfigProvider) Read(object string, field string, localcontext interface{}) (interface{}, error) {
+	clog.Debug(" M (Catalog): Read, object: %s, field: %s", object, field)
 	namespace := m.getNamespaceFromContext(localcontext)
-
+	object = api_utils.ReplaceSeperator(object)
 	catalog, err := m.ApiClient.GetCatalog(context.TODO(), object, namespace, m.Config.User, m.Config.Password)
 	if err != nil {
 		return "", err
@@ -128,7 +133,9 @@ func (m *CatalogConfigProvider) Read(object string, field string, localcontext i
 }
 
 func (m *CatalogConfigProvider) ReadObject(object string, localcontext interface{}) (map[string]interface{}, error) {
+	clog.Debug(" M (Catalog): ReadObject, object: %s", object)
 	namespace := m.getNamespaceFromContext(localcontext)
+	object = api_utils.ReplaceSeperator(object)
 
 	catalog, err := m.ApiClient.GetCatalog(context.TODO(), object, namespace, m.Config.User, m.Config.Password)
 	if err != nil {
@@ -221,6 +228,7 @@ func (m *CatalogConfigProvider) traceValue(v interface{}, localcontext interface
 // TODO: IConfigProvider interface methods should be enhanced to accept namespace as a parameter
 // so we can get rid of getCatalogInDefaultNamespace.
 func (m *CatalogConfigProvider) Set(object string, field string, value interface{}) error {
+	clog.Debug(" M (Catalog): Set, object: %s, field: %s", object, field)
 	catalog, err := m.getCatalogInDefaultNamespace(context.TODO(), object)
 	if err != nil {
 		return err
@@ -230,6 +238,7 @@ func (m *CatalogConfigProvider) Set(object string, field string, value interface
 	return m.ApiClient.UpsertCatalog(context.TODO(), object, data, m.Config.User, m.Config.Password)
 }
 func (m *CatalogConfigProvider) SetObject(object string, value map[string]interface{}) error {
+	clog.Debug(" M (Catalog): SetObject, object: %s", object)
 	catalog, err := m.getCatalogInDefaultNamespace(context.TODO(), object)
 	if err != nil {
 		return err
@@ -242,6 +251,7 @@ func (m *CatalogConfigProvider) SetObject(object string, value map[string]interf
 	return m.ApiClient.UpsertCatalog(context.TODO(), object, data, m.Config.User, m.Config.Password)
 }
 func (m *CatalogConfigProvider) Remove(object string, field string) error {
+	clog.Debug(" M (Catalog): Remove, object: %s, field: %s", object, field)
 	catlog, err := m.getCatalogInDefaultNamespace(context.TODO(), object)
 	if err != nil {
 		return err
@@ -254,9 +264,12 @@ func (m *CatalogConfigProvider) Remove(object string, field string) error {
 	return m.ApiClient.UpsertCatalog(context.TODO(), object, data, m.Config.User, m.Config.Password)
 }
 func (m *CatalogConfigProvider) RemoveObject(object string) error {
+	clog.Debug(" M (Catalog): RemoveObject, object: %s", object)
+	object = api_utils.ReplaceSeperator(object)
 	return m.ApiClient.DeleteCatalog(context.TODO(), object, m.Config.User, m.Config.Password)
 }
 
 func (m *CatalogConfigProvider) getCatalogInDefaultNamespace(context context.Context, catalog string) (model.CatalogState, error) {
+	catalog = api_utils.ReplaceSeperator(catalog)
 	return m.ApiClient.GetCatalog(context, catalog, "", m.Config.User, m.Config.Password)
 }

api/pkg/apis/v1alpha1/providers/config/catalog/catalogprovider_test.go

@@ -44,13 +44,13 @@ func TestRead(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		var response interface{}
 		switch r.URL.Path {
-		case "/catalogs/registry/catalog1":
+		case "/catalogs/registry/catalog1-v1":
 			response = model.CatalogState{
 				ObjectMeta: model.ObjectMeta{
-					Name: "catalog1",
+					Name: "catalog1-v1",
 				},
 				Spec: &model.CatalogSpec{
-					ParentName: "parent",
+					ParentName: "parent:v1",
 					Properties: map[string]interface{}{
 						"components": []model.ComponentSpec{
 							{
@@ -61,10 +61,10 @@ func TestRead(t *testing.T) {
 					},
 				},
 			}
-		case "/catalogs/registry/parent":
+		case "/catalogs/registry/parent-v1":
 			response = model.CatalogState{
 				ObjectMeta: model.ObjectMeta{
-					Name: "parent",
+					Name: "parent-v1",
 				},
 				Spec: &model.CatalogSpec{
 					Properties: map[string]interface{}{
@@ -94,7 +94,7 @@ func TestRead(t *testing.T) {
 	}
 	assert.Nil(t, err)
 
-	res, err := provider.Read("catalog1", "components", nil)
+	res, err := provider.Read("catalog1:v1", "components", nil)
 	assert.Nil(t, err)
 	data, err := json.Marshal(res)
 	assert.Nil(t, err)
@@ -103,13 +103,13 @@ func TestRead(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, "name", summary[0].Name)
 
-	res, err = provider.Read("catalog1", "parentAttribute", nil)
+	res, err = provider.Read("catalog1:v1", "parentAttribute", nil)
 	assert.Nil(t, err)
 	v, ok := res.(string)
 	assert.True(t, ok)
 	assert.Equal(t, "This is father", v)
 
-	res, err = provider.Read("catalog1", "notExist", nil)
+	res, err = provider.Read("catalog1:v1", "notExist", nil)
 	coaErr := err.(v1alpha2.COAError)
 	assert.Equal(t, v1alpha2.NotFound, coaErr.State)
 	assert.Empty(t, res)
@@ -119,13 +119,13 @@ func TestReadObject(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		var response interface{}
 		switch r.URL.Path {
-		case "/catalogs/registry/catalog1":
+		case "/catalogs/registry/catalog1-v1":
 			response = model.CatalogState{
 				ObjectMeta: model.ObjectMeta{
-					Name: "catalog1",
+					Name: "catalog1-v1",
 				},
 				Spec: &model.CatalogSpec{
-					ParentName: "parent",
+					ParentName: "parent:v1",
 					Properties: map[string]interface{}{
 						"components": map[string]interface{}{
 							"Name": "name",
@@ -134,10 +134,10 @@ func TestReadObject(t *testing.T) {
 					},
 				},
 			}
-		case "/catalogs/registry/parent":
+		case "/catalogs/registry/parent-v1":
 			response = model.CatalogState{
 				ObjectMeta: model.ObjectMeta{
-					Name: "parent",
+					Name: "parent-v1",
 				},
 				Spec: &model.CatalogSpec{
 					Properties: map[string]interface{}{
@@ -167,7 +167,7 @@ func TestReadObject(t *testing.T) {
 	}
 	assert.Nil(t, err)
 
-	res, err := provider.ReadObject("catalog1", nil)
+	res, err := provider.ReadObject("catalog1:v1", nil)
 	assert.Nil(t, err)
 	assert.Equal(t, "name", res["Name"])
 }

api/pkg/apis/v1alpha1/providers/stage/create/create.go

@@ -18,6 +18,7 @@ import (
 	"github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/model"
 	"github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/providers/stage"
 	"github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/utils"
+	api_utils "github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/utils"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2/contexts"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2/observability"
@@ -135,6 +136,7 @@ func (i *CreateStageProvider) Process(ctx context.Context, mgrContext contexts.M
 	if object != nil {
 		oData, _ = json.Marshal(object)
 	}
+	objectName = api_utils.ReplaceSeperator(objectName)
 	lastSummaryMessage := ""
 	switch objectType {
 	case "instance":

api/pkg/apis/v1alpha1/providers/stage/materialize/materialize.go

@@ -16,6 +16,7 @@ import (
 	"github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/model"
 	"github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/providers/stage"
 	"github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/utils"
+	api_utils "github.com/eclipse-symphony/symphony/api/pkg/apis/v1alpha1/utils"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2/contexts"
 	"github.com/eclipse-symphony/symphony/coa/pkg/apis/v1alpha2/observability"
@@ -105,7 +106,8 @@ func (i *MaterializeStageProvider) Process(ctx context.Context, mgrContext conte
 	})
 	var err error = nil
 	defer observ_utils.CloseSpanWithError(span, &err)
-	mLog.Info("  P (Materialize Processor): processing inputs")
+	mLog.Infof("  P (Materialize Processor): processing inputs, traceId: %s", span.SpanContext().TraceID().String())
+
 	outputs := make(map[string]interface{})
 
 	objects, ok := inputs["names"].([]interface{})
@@ -142,6 +144,7 @@ func (i *MaterializeStageProvider) Process(ctx context.Context, mgrContext conte
 	creationCount := 0
 	for _, catalog := range catalogs {
 		for _, object := range prefixedNames {
+			object := api_utils.ReplaceSeperator(object)
 			if catalog.ObjectMeta.Name == object {
 				objectData, _ := json.Marshal(catalog.Spec.Properties) //TODO: handle errors
 				name := catalog.ObjectMeta.Name