Add get,set,remove functions for config sections using redis + tests

engine/datadbmock.go

@@ -446,3 +446,15 @@ func (dbM *DataDBMock) SetVersions(vrs Versions, overwrite bool) (err error) {
 func (dbM *DataDBMock) RemoveRatingProfileDrv(string) error {
 	return utils.ErrNotImplemented
 }
+
+func (dbM *DataDBMock) GetConfigSectionsDrv(ctx *context.Context, tenant, nodeID string, sectionIDs []string) (map[string][]byte, error) {
+	return nil, utils.ErrNotImplemented
+}
+
+func (dbM *DataDBMock) SetConfigSectionsDrv(ctx *context.Context, tenant, nodeID string, sectionsData map[string][]byte) error {
+	return utils.ErrNotImplemented
+}
+
+func (dbM *DataDBMock) RemoveConfigSectionsDrv(ctx *context.Context, tenant, nodeID string, sectionIDs []string) error {
+	return utils.ErrNotImplemented
+}

engine/storage_interface.go

@@ -95,6 +95,9 @@ type DataDB interface {
 	GetAccountDrv(*context.Context, string, string) (*utils.Account, error)
 	SetAccountDrv(ctx *context.Context, profile *utils.Account) error
 	RemoveAccountDrv(*context.Context, string, string) error
+	GetConfigSectionsDrv(*context.Context, string, string, []string) (map[string][]byte, error)
+	SetConfigSectionsDrv(*context.Context, string, string, map[string][]byte) error
+	RemoveConfigSectionsDrv(*context.Context, string, string, []string) error
 }
 
 // DataDBDriver used as a DataDB but also as a ConfigProvider

engine/storage_internal_datadb.go

@@ -651,3 +651,15 @@ func (iDB *InternalDB) RemoveAccountDrv(_ *context.Context, tenant, id string) (
 		true, utils.NonTransactional)
 	return
 }
+
+func (iDB *InternalDB) GetConfigSectionsDrv(ctx *context.Context, tenant, nodeID string, sectionIDs []string) (map[string][]byte, error) {
+	return nil, utils.ErrNotImplemented
+}
+
+func (iDB *InternalDB) SetConfigSectionsDrv(ctx *context.Context, tenant, nodeID string, sectionsData map[string][]byte) error {
+	return utils.ErrNotImplemented
+}
+
+func (iDB *InternalDB) RemoveConfigSectionsDrv(ctx *context.Context, tenant, nodeID string, sectionIDs []string) error {
+	return utils.ErrNotImplemented
+}

engine/storage_mongo_datadb.go

@@ -1497,6 +1497,18 @@ func (ms *MongoStorage) RemoveAccountDrv(ctx *context.Context, tenant, id string
 	})
 }
 
+func (ms *MongoStorage) GetConfigSectionsDrv(ctx *context.Context, tenant, nodeID string, sectionIDs []string) (map[string][]byte, error) {
+	return nil, utils.ErrNotImplemented
+}
+
+func (ms *MongoStorage) SetConfigSectionsDrv(ctx *context.Context, tenant, nodeID string, sectionsData map[string][]byte) error {
+	return utils.ErrNotImplemented
+}
+
+func (ms *MongoStorage) RemoveConfigSectionsDrv(ctx *context.Context, tenant, nodeID string, sectionIDs []string) error {
+	return utils.ErrNotImplemented
+}
+
 func newAggregateStages(profileID, tenant, prefix string) (match, query bson.D) {
 	match = bson.D{{
 		"$match", bson.M{

engine/storage_redis.go

@@ -934,3 +934,35 @@ func (rs *RedisStorage) SetAccountDrv(ctx *context.Context, ap *utils.Account) (
 func (rs *RedisStorage) RemoveAccountDrv(ctx *context.Context, tenant, id string) (err error) {
 	return rs.Cmd(nil, redisDEL, utils.AccountPrefix+utils.ConcatenatedKey(tenant, id))
 }
+
+func (rs *RedisStorage) GetConfigSectionsDrv(ctx *context.Context, tenant, nodeID string, sectionIDs []string) (sectionMap map[string][]byte, err error) {
+	sectionMap = make(map[string][]byte)
+	for _, sectionID := range sectionIDs {
+		var value []byte
+		if err = rs.Cmd(&value, redisGET, utils.ConfigPrefix+sectionID); err != nil {
+			return
+		}
+		if value != nil {
+			sectionMap[sectionID] = value
+		}
+	}
+	return
+}
+
+func (rs *RedisStorage) SetConfigSectionsDrv(ctx *context.Context, tenant, nodeID string, sectionsData map[string][]byte) (err error) {
+	for sectionID, sectionData := range sectionsData {
+		if err = rs.Cmd(nil, redisSET, utils.ConfigPrefix+sectionID, string(sectionData)); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (rs *RedisStorage) RemoveConfigSectionsDrv(ctx *context.Context, tenant, nodeID string, sectionIDs []string) (err error) {
+	for _, sectionID := range sectionIDs {
+		if err = rs.Cmd(nil, redisDEL, utils.ConfigPrefix+sectionID); err != nil {
+			return
+		}
+	}
+	return
+}

engine/storage_redis_config.go

@@ -20,15 +20,12 @@ package engine
 
 import (
 	"github.com/cgrates/birpc/context"
-)
-
-const (
-	configPrefix = "cfg_"
+	"github.com/cgrates/cgrates/utils"
 )
 
 func (rs *RedisStorage) GetSection(ctx *context.Context, section string, val interface{}) (err error) {
 	var values []byte
-	if err = rs.Cmd(&values, redisGET, configPrefix+section); err != nil || len(values) == 0 {
+	if err = rs.Cmd(&values, redisGET, utils.ConfigPrefix+section); err != nil || len(values) == 0 {
 		return
 	}
 	err = rs.ms.Unmarshal(values, val)
@@ -40,5 +37,5 @@ func (rs *RedisStorage) SetSection(_ *context.Context, section string, jsn inter
 	if result, err = rs.ms.Marshal(jsn); err != nil {
 		return
 	}
-	return rs.Cmd(nil, redisSET, configPrefix+section, string(result))
+	return rs.Cmd(nil, redisSET, utils.ConfigPrefix+section, string(result))
 }

engine/storage_redis_it_test.go

@@ -0,0 +1,150 @@
+//go:build integration
+// +build integration
+
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package engine
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/cgrates/birpc/context"
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func TestSetGetRemoveConfigSectionsDrv(t *testing.T) {
+	cfg := config.NewDefaultCGRConfig()
+	db, err := NewRedisStorage(cfg.DataDbCfg().Host+":"+cfg.DataDbCfg().Port, 10, cfg.DataDbCfg().User,
+		cfg.DataDbCfg().Password, cfg.GeneralCfg().DBDataEncoding, utils.RedisMaxConns, utils.RedisMaxAttempts,
+		utils.EmptyString, false, 0, 0, false, utils.EmptyString, utils.EmptyString, utils.EmptyString)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	sectionIDs := []string{"thresholds", "resources"}
+	expected := make(map[string][]byte)
+
+	// Try to retrieve the values before setting them (should receive an empty map)
+	if rcv, err := db.GetConfigSectionsDrv(context.Background(), "cgrates.org", "1234", sectionIDs); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(rcv, expected) {
+		t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expected), utils.ToJSON(rcv))
+	}
+
+	ms, err := utils.NewMarshaler(utils.JSON)
+	if err != nil {
+		t.Fatal(err)
+	}
+	thCfg := &config.ThresholdSJsonCfg{
+		Enabled:               utils.BoolPointer(true),
+		Indexed_selects:       utils.BoolPointer(true),
+		Store_interval:        utils.StringPointer("2s"),
+		String_indexed_fields: &[]string{"req.index11"},
+		Prefix_indexed_fields: &[]string{"req.index22"},
+		Suffix_indexed_fields: &[]string{"req.index33"},
+		Actions_conns:         &[]string{"*internal"},
+		Nested_fields:         utils.BoolPointer(true),
+		Opts: &config.ThresholdsOptsJson{
+			ProfileIDs: []*utils.DynamicStringSliceOpt{
+				{
+					Tenant: "cgrates.org",
+					Value:  []string{"value1"},
+				},
+			},
+			ProfileIgnoreFilters: []*utils.DynamicBoolOpt{
+				{
+					Tenant: "cgrates.org",
+					Value:  true,
+				},
+			},
+		},
+	}
+	thJsnCfg, err := ms.Marshal(thCfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rsCfg := &config.ResourceSJsonCfg{
+		Enabled:               utils.BoolPointer(true),
+		Indexed_selects:       utils.BoolPointer(true),
+		Thresholds_conns:      &[]string{"*birpc"},
+		Store_interval:        utils.StringPointer("2s"),
+		String_indexed_fields: &[]string{"*req.index11"},
+		Prefix_indexed_fields: &[]string{"*req.index22"},
+		Suffix_indexed_fields: &[]string{"*req.index33"},
+		Nested_fields:         utils.BoolPointer(true),
+		Opts: &config.ResourcesOptsJson{
+			UsageID: []*utils.DynamicStringOpt{
+				{
+					Value: "usg2",
+				},
+			},
+			UsageTTL: []*utils.DynamicStringOpt{
+				{
+					Value: "1m0s",
+				},
+			},
+			Units: []*utils.DynamicFloat64Opt{
+				{
+					Value: 2,
+				},
+			},
+		},
+	}
+	rsJsnCfg, err := ms.Marshal(rsCfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sectData := map[string][]byte{
+		"thresholds": thJsnCfg,
+		"resources":  rsJsnCfg,
+	}
+
+	if err := db.SetConfigSectionsDrv(context.Background(), "cgrates.org", "1234", sectData); err != nil {
+		t.Fatal(err)
+	}
+
+	if rcv, err := db.GetConfigSectionsDrv(context.Background(), "cgrates.org", "1234", sectionIDs); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(rcv, sectData) {
+		t.Fatalf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(sectData), utils.ToJSON(rcv))
+	} else {
+		rcvThCfg := &config.ThresholdSJsonCfg{}
+		ms.Unmarshal(rcv["thresholds"], &rcvThCfg)
+		if !reflect.DeepEqual(rcvThCfg, thCfg) {
+			t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(thCfg), utils.ToJSON(rcvThCfg))
+		}
+		rcvRsCfg := &config.ResourceSJsonCfg{}
+		ms.Unmarshal(rcv["resources"], &rcvRsCfg)
+		if !reflect.DeepEqual(rcvRsCfg, rsCfg) {
+			t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(rsCfg), utils.ToJSON(rcvRsCfg))
+		}
+	}
+
+	if err := db.RemoveConfigSectionsDrv(context.Background(), "cgrates.org", "1234", sectionIDs); err != nil {
+		t.Fatal(err)
+	}
+
+	if rcv, err := db.GetConfigSectionsDrv(context.Background(), "cgrates.org", "1234", sectionIDs); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(rcv, expected) {
+		t.Errorf("expected: <%+v>, \nreceived: <%+v>", utils.ToJSON(expected), utils.ToJSON(rcv))
+	}
+}

utils/consts.go

@@ -261,6 +261,7 @@ const (
 
 	MetaVoice                 = "*voice"
 	ACD                       = "ACD"
+	ConfigPrefix              = "cfg_"
 	ResourcesPrefix           = "res_"
 	ResourceProfilesPrefix    = "rsp_"
 	ThresholdPrefix           = "thd_"