Added migration for DerivedChargers to ChargersProfile and tests

engine/version.go

@@ -152,7 +152,7 @@ func CurrentDataDBVersions() Versions {
 		utils.Alias:               2,
 		utils.User:                2,
 		utils.Subscribers:         1,
-		utils.DerivedChargersV:    1,
+		utils.DerivedChargersV:    2,
 		utils.Destinations:        1,
 		utils.ReverseDestinations: 1,
 		utils.RatingPlan:          1,

migrator/derived_chargers.go

@@ -22,10 +22,198 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/cgrates/cgrates/config"
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
+var dcGetMapKeys = func(m utils.StringMap) (keys []string) {
+	keys = make([]string, len(m))
+	i := 0
+	for k, _ := range m {
+		keys[i] = k
+		i += 1
+	}
+	// sort.Strings(keys)
+	return keys
+}
+
+type v1DerivedCharger struct {
+	RunID      string // Unique runId in the chain
+	RunFilters string // Only run the charger if all the filters match
+
+	RequestTypeField     string // Field containing request type info, number in case of csv source, '^' as prefix in case of static values
+	DirectionField       string // Field containing direction info
+	TenantField          string // Field containing tenant info
+	CategoryField        string // Field containing tor info
+	AccountField         string // Field containing account information
+	SubjectField         string // Field containing subject information
+	DestinationField     string // Field containing destination information
+	SetupTimeField       string // Field containing setup time information
+	PDDField             string // Field containing setup time information
+	AnswerTimeField      string // Field containing answer time information
+	UsageField           string // Field containing usage information
+	SupplierField        string // Field containing supplier information
+	DisconnectCauseField string // Field containing disconnect cause information
+	CostField            string // Field containing cost information
+	PreRatedField        string // Field marking rated request in CDR
+}
+
+type v1DerivedChargers struct {
+	DestinationIDs utils.StringMap
+	Chargers       []*v1DerivedCharger
+}
+
+type v1DerivedChargersWithKey struct {
+	Key   string
+	Value *v1DerivedChargers
+}
+
+func fieldinfo2Attribute(attr []*engine.Attribute, fieldName, fieldInfo string) (a []*engine.Attribute) {
+	var rp config.RSRParsers
+	if fieldInfo == utils.META_DEFAULT || len(fieldInfo) == 0 {
+		return attr
+	}
+	if strings.HasPrefix(fieldInfo, utils.STATIC_VALUE_PREFIX) {
+		fieldInfo = fieldInfo[1:]
+	}
+	var err error
+	if rp, err = config.NewRSRParsers(fieldInfo, true, utils.INFIELD_SEP); err != nil {
+		utils.Logger.Err(fmt.Sprintf("On Migrating rule: <%s>, error: %s", fieldInfo, err.Error()))
+		return attr
+	}
+	return append(attr, &engine.Attribute{
+		FieldName:  fieldName,
+		Initial:    utils.META_ANY,
+		Substitute: rp,
+		Append:     true,
+	})
+}
+func derivedChargers2AttributeProfile(dc *v1DerivedCharger, tenant, key string, filters []string) (attr *engine.AttributeProfile) {
+	attr = &engine.AttributeProfile{
+		Tenant:             tenant,
+		ID:                 key,
+		Contexts:           []string{utils.META_ANY},
+		FilterIDs:          filters,
+		ActivationInterval: nil,
+		Attributes:         make([]*engine.Attribute, 0),
+		Blocker:            false,
+		Weight:             10,
+	}
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.RequestType, dc.RequestTypeField)
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.Direction, dc.DirectionField) //still in use?
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.Tenant, dc.TenantField)
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.Category, dc.CategoryField)
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.Account, dc.AccountField)
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.Subject, dc.SubjectField)
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.Destination, dc.DestinationField)
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.SetupTime, dc.SetupTimeField)
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.PDD, dc.PDDField)
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.AnswerTime, dc.AnswerTimeField)
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.Usage, dc.UsageField)
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.SUPPLIER, dc.SupplierField)
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.DISCONNECT_CAUSE, dc.DisconnectCauseField)
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.Cost, dc.CostField)
+	attr.Attributes = fieldinfo2Attribute(attr.Attributes, utils.PreRated, dc.PreRatedField)
+	return
+}
+
+func derivedChargers2Charger(dc *v1DerivedCharger, tenant string, key string, filters []string) (ch *engine.ChargerProfile) {
+	ch = &engine.ChargerProfile{
+		Tenant:             tenant,
+		ID:                 key,
+		FilterIDs:          filters,
+		ActivationInterval: nil,
+		RunID:              dc.RunID,
+		AttributeIDs:       make([]string, 0),
+		Weight:             10,
+	}
+
+	filter := dc.RunFilters
+	if len(filter) != 0 {
+		if strings.HasPrefix(filter, utils.STATIC_VALUE_PREFIX) {
+			filter = filter[1:]
+		}
+		ch.FilterIDs = append(ch.FilterIDs, "*rsr::"+filter)
+	}
+	return
+}
+
+func (m *Migrator) derivedChargers2Chargers(dck *v1DerivedChargersWithKey) (err error) {
+	// (direction, tenant, category, account, subject)
+	skey := utils.SplitConcatenatedKey(dck.Key)
+	destination := ""
+	if len(dck.Value.DestinationIDs) != 0 {
+		destination = "*destination:Destination:"
+		keys := dcGetMapKeys(dck.Value.DestinationIDs)
+		destination += strings.Join(keys, utils.INFIELD_SEP)
+	}
+	filter := make([]string, 0)
+
+	if len(destination) != 0 {
+		filter = append(filter, destination)
+	}
+	if len(skey[2]) != 0 && skey[2] != utils.META_ANY {
+		filter = append(filter, "*string:Category:"+skey[2])
+	}
+	if len(skey[3]) != 0 && skey[3] != utils.META_ANY {
+		filter = append(filter, "*string:Account:"+skey[3])
+	}
+	if len(skey[4]) != 0 && skey[4] != utils.META_ANY {
+		filter = append(filter, "*string:Subject:"+skey[4])
+	}
+
+	for i, dc := range dck.Value.Chargers {
+		attr := derivedChargers2AttributeProfile(dc, skey[1], fmt.Sprintf("%s%v", dck.Key, i), filter)
+		ch := derivedChargers2Charger(dc, skey[1], fmt.Sprintf("%s%v", dck.Key, i), filter)
+		if len(attr.Attributes) != 0 {
+			if err = m.dmOut.DataManager().DataDB().SetAttributeProfileDrv(attr); err != nil {
+				return err
+			}
+			ch.AttributeIDs = append(ch.AttributeIDs, attr.ID)
+		}
+		if err = m.dmOut.DataManager().DataDB().SetChargerProfileDrv(ch); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (m *Migrator) migrateV1DerivedChargers() (err error) {
+	for {
+		var dck *v1DerivedChargersWithKey
+		dck, err = m.dmIN.getV1DerivedChargers()
+		if err == utils.ErrNoMoreData {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		if dck == nil || m.dryRun {
+			continue
+		}
+		if err = m.derivedChargers2Chargers(dck); err != nil {
+			return err
+		}
+		if err = m.dmIN.remV1DerivedChargers(dck.Key); err != nil {
+			return err
+		}
+		m.stats[utils.DerivedChargersV] += 1
+	}
+	if m.dryRun {
+		return
+	}
+	// All done, update version wtih current one
+	vrs := engine.Versions{utils.DerivedChargersV: engine.CurrentDataDBVersions()[utils.DerivedChargersV]}
+	if err = m.dmOut.DataManager().DataDB().SetVersions(vrs, false); err != nil {
+		return utils.NewCGRError(utils.Migrator,
+			utils.ServerErrorCaps,
+			err.Error(),
+			fmt.Sprintf("error: <%s> when updating DerivedChargers version into dataDB", err.Error()))
+	}
+	return
+}
+
 func (m *Migrator) migrateCurrentDerivedChargers() (err error) {
 	var ids []string
 	ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.DERIVEDCHARGERS_PREFIX)
@@ -63,9 +251,12 @@ func (m *Migrator) migrateDerivedChargers() (err error) {
 		return utils.NewCGRError(utils.Migrator,
 			utils.MandatoryIEMissingCaps,
 			utils.UndefinedVersion,
-			"version number is not defined for ActionTriggers model")
+			"version number is not defined for DerivedChargers model")
 	}
+
 	switch vrs[utils.DerivedChargersV] {
+	case 1:
+		return m.migrateV1DerivedChargers()
 	case current[utils.DerivedChargersV]:
 		if m.sameDataDB {
 			return

migrator/derived_chargers_it_test.go

@@ -0,0 +1,286 @@
+// +build integration
+
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package migrator
+
+import (
+	"fmt"
+	"log"
+	"path"
+	"reflect"
+	"sort"
+	"testing"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var (
+	dcCfgIn    *config.CGRConfig
+	dcCfgOut   *config.CGRConfig
+	dcMigrator *Migrator
+	dcAction   string
+)
+
+var sTestsDCIT = []func(t *testing.T){
+	testDCITConnect,
+	testDCITFlush,
+	testDCITMigrateAndMove,
+}
+
+func TestDerivedChargersVMigrateITRedis(t *testing.T) {
+	inPath := path.Join(*dataDir, "conf", "samples", "tutmysql")
+	testStartDC("TestDerivedChargersVMigrateITRedis", inPath, inPath, utils.Migrate, t)
+}
+
+func TestDerivedChargersVMigrateITMongo(t *testing.T) {
+	inPath := path.Join(*dataDir, "conf", "samples", "tutmongo")
+	testStartDC("TestDerivedChargersVMigrateITMongo", inPath, inPath, utils.Migrate, t)
+}
+
+func TestDerivedChargersVITMove(t *testing.T) {
+	inPath := path.Join(*dataDir, "conf", "samples", "tutmongo")
+	outPath := path.Join(*dataDir, "conf", "samples", "tutmysql")
+	testStartDC("TestDerivedChargersVITMove", inPath, outPath, utils.Move, t)
+}
+
+func TestDerivedChargersVITMigrateMongo2Redis(t *testing.T) {
+	inPath := path.Join(*dataDir, "conf", "samples", "tutmongo")
+	outPath := path.Join(*dataDir, "conf", "samples", "tutmysql")
+	testStartDC("TestDerivedChargersVITMigrateMongo2Redis", inPath, outPath, utils.Migrate, t)
+}
+
+func TestDerivedChargersVITMoveEncoding(t *testing.T) {
+	inPath := path.Join(*dataDir, "conf", "samples", "tutmongo")
+	outPath := path.Join(*dataDir, "conf", "samples", "tutmongojson")
+	testStartDC("TestDerivedChargersVITMoveEncoding", inPath, outPath, utils.Move, t)
+}
+
+func TestDerivedChargersVITMoveEncoding2(t *testing.T) {
+	inPath := path.Join(*dataDir, "conf", "samples", "tutmysql")
+	outPath := path.Join(*dataDir, "conf", "samples", "tutmysqljson")
+	testStartDC("TestDerivedChargersVITMoveEncoding2", inPath, outPath, utils.Move, t)
+}
+
+func testStartDC(testName, inPath, outPath, action string, t *testing.T) {
+	var err error
+	dcAction = action
+	if dcCfgIn, err = config.NewCGRConfigFromFolder(inPath); err != nil {
+		t.Fatal(err)
+	}
+	if dcCfgOut, err = config.NewCGRConfigFromFolder(outPath); err != nil {
+		t.Fatal(err)
+	}
+	for _, stest := range sTestsDCIT {
+		t.Run(testName, stest)
+	}
+}
+
+func testDCITConnect(t *testing.T) {
+	dataDBIn, err := NewMigratorDataDB(dcCfgIn.DataDbCfg().DataDbType,
+		dcCfgIn.DataDbCfg().DataDbHost, dcCfgIn.DataDbCfg().DataDbPort,
+		dcCfgIn.DataDbCfg().DataDbName, dcCfgIn.DataDbCfg().DataDbUser,
+		dcCfgIn.DataDbCfg().DataDbPass, dcCfgIn.GeneralCfg().DBDataEncoding,
+		config.CgrConfig().CacheCfg(), "")
+	if err != nil {
+		log.Fatal(err)
+	}
+	dataDBOut, err := NewMigratorDataDB(dcCfgOut.DataDbCfg().DataDbType,
+		dcCfgOut.DataDbCfg().DataDbHost, dcCfgOut.DataDbCfg().DataDbPort,
+		dcCfgOut.DataDbCfg().DataDbName, dcCfgOut.DataDbCfg().DataDbUser,
+		dcCfgOut.DataDbCfg().DataDbPass, dcCfgOut.GeneralCfg().DBDataEncoding,
+		config.CgrConfig().CacheCfg(), "")
+	if err != nil {
+		log.Fatal(err)
+	}
+	dcMigrator, err = NewMigrator(dataDBIn, dataDBOut,
+		nil, nil, false, false, false)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func testDCITFlush(t *testing.T) {
+	dcMigrator.dmOut.DataManager().DataDB().Flush("")
+	if err := engine.SetDBVersions(dcMigrator.dmOut.DataManager().DataDB()); err != nil {
+		t.Error("Error  ", err.Error())
+	}
+	dcMigrator.dmIN.DataManager().DataDB().Flush("")
+	if err := engine.SetDBVersions(dcMigrator.dmIN.DataManager().DataDB()); err != nil {
+		t.Error("Error  ", err.Error())
+	}
+}
+
+func testDCITMigrateAndMove(t *testing.T) {
+	dcGetMapKeys = func(m utils.StringMap) (keys []string) { //make sure destination are in order
+		keys = make([]string, len(m))
+		i := 0
+		for k, _ := range m {
+			keys[i] = k
+			i += 1
+		}
+		sort.Strings(keys)
+		return keys
+	}
+	derivch := &v1DerivedChargersWithKey{
+		Key: utils.ConcatenatedKey("*out", defaultTenant, utils.META_ANY, "1003", utils.META_ANY),
+		Value: &v1DerivedChargers{
+			DestinationIDs: utils.StringMap{"1001": true, "1002": true, "1003": true},
+			Chargers: []*v1DerivedCharger{
+				&v1DerivedCharger{
+					RunID:      "RunID",
+					RunFilters: "~filterhdr1:s/(.+)/special_run3/",
+
+					RequestTypeField: utils.MetaDefault,
+					CategoryField:    utils.MetaDefault,
+					AccountField:     "^1004",
+					SubjectField:     "call_1003",
+				},
+			},
+		},
+	}
+	attrProf := &engine.AttributeProfile{
+		Tenant:   "cgrates.org",
+		ID:       fmt.Sprintf("%s%v", derivch.Key, 0),
+		Contexts: []string{utils.META_ANY},
+		FilterIDs: []string{
+			"*destination:Destination:1001;1002;1003",
+			"*string:Account:1003",
+		},
+		ActivationInterval: nil,
+		Attributes: []*engine.Attribute{
+			{
+				FieldName:  utils.Account,
+				Initial:    utils.META_ANY,
+				Substitute: config.NewRSRParsersMustCompile("1004", true, utils.INFIELD_SEP),
+				Append:     true,
+			},
+			{
+				FieldName:  utils.Subject,
+				Initial:    utils.META_ANY,
+				Substitute: config.NewRSRParsersMustCompile("call_1003", true, utils.INFIELD_SEP),
+				Append:     true,
+			},
+		},
+		Blocker: false,
+		Weight:  10,
+	}
+	attrProf.Compile()
+	charger := &engine.ChargerProfile{
+		Tenant: defaultTenant,
+		ID:     fmt.Sprintf("%s%v", derivch.Key, 0),
+		FilterIDs: []string{
+			"*destination:Destination:1001;1002;1003",
+			"*string:Account:1003",
+			"*rsr::~filterhdr1:s/(.+)/special_run3/",
+		},
+		ActivationInterval: nil,
+		RunID:              "RunID",
+		AttributeIDs:       []string{attrProf.ID},
+		Weight:             10,
+	}
+	switch dcAction {
+	case utils.Migrate:
+		err := dcMigrator.dmIN.setV1DerivedChargers(derivch)
+		if err != nil {
+			t.Error("Error when setting v1 DerivedChargersV ", err.Error())
+		}
+		currentVersion := engine.Versions{utils.DerivedChargersV: 1}
+		err = dcMigrator.dmIN.DataManager().DataDB().SetVersions(currentVersion, false)
+		if err != nil {
+			t.Error("Error when setting version for DerivedChargersV ", err.Error())
+		}
+		//check if version was set correctly
+		if vrs, err := dcMigrator.dmIN.DataManager().DataDB().GetVersions(""); err != nil {
+			t.Error(err)
+		} else if vrs[utils.DerivedChargersV] != 1 {
+			t.Errorf("Unexpected version returned: %d", vrs[utils.DerivedChargersV])
+		}
+		//migrate derivch
+		err, _ = dcMigrator.Migrate([]string{utils.MetaDerivedChargersV})
+		if err != nil {
+			t.Error("Error when migrating DerivedChargersV ", err.Error())
+		}
+		//check if version was updated
+		if vrs, err := dcMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil {
+			t.Error(err)
+		} else if vrs[utils.DerivedChargersV] != 2 {
+			t.Errorf("Unexpected version returned: %d", vrs[utils.DerivedChargersV])
+		}
+		//check if derivch was migrate correctly
+		result, err := dcMigrator.dmOut.DataManager().DataDB().GetAttributeProfileDrv(defaultTenant, attrProf.ID)
+		if err != nil {
+			t.Fatalf("Error when getting Attributes %v", err.Error())
+		}
+		result.Compile()
+		sort.Slice(result.Attributes, func(i, j int) bool {
+			if result.Attributes[i].FieldName == result.Attributes[j].FieldName {
+				return result.Attributes[i].Initial.(string) < result.Attributes[j].Initial.(string)
+			}
+			return result.Attributes[i].FieldName < result.Attributes[j].FieldName
+		}) // only for test; map returns random keys
+		if !reflect.DeepEqual(*attrProf, *result) {
+			t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(attrProf), utils.ToJSON(result))
+		}
+		result2, err := dcMigrator.dmOut.DataManager().DataDB().GetChargerProfileDrv(defaultTenant, charger.ID)
+		if err != nil {
+			t.Fatalf("Error when getting Attributes %v", err.Error())
+		}
+		if !reflect.DeepEqual(*charger, *result2) {
+			t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(charger), utils.ToJSON(result2))
+		}
+
+		//check if old account was deleted
+		if _, err = dcMigrator.dmIN.getV1DerivedChargers(); err != utils.ErrNoMoreData {
+			t.Error("Error should be not found : ", err)
+		}
+
+	case utils.Move:
+		/* // No Move tests
+		if err := dcMigrator.dmIN.DataManager().DataDB().SetDerivedChargersV(derivch, utils.NonTransactional); err != nil {
+			t.Error(err)
+		}
+		currentVersion := engine.CurrentDataDBVersions()
+		err := dcMigrator.dmOut.DataManager().DataDB().SetVersions(currentVersion, false)
+		if err != nil {
+			t.Error("Error when setting version for DerivedChargersV ", err.Error())
+		}
+		//migrate accounts
+		err, _ = dcMigrator.Migrate([]string{utils.MetaDerivedChargersV})
+		if err != nil {
+			t.Error("Error when dcMigratorrating DerivedChargersV ", err.Error())
+		}
+		//check if account was migrate correctly
+		result, err := dcMigrator.dmOut.DataManager().DataDB().GetDerivedChargersV(derivch.GetId(), false)
+		if err != nil {
+			t.Error(err)
+		}
+		if !reflect.DeepEqual(derivch, result) {
+			t.Errorf("Expecting: %+v, received: %+v", derivch, result)
+		}
+		//check if old account was deleted
+		result, err = dcMigrator.dmIN.DataManager().DataDB().GetDerivedChargersV(derivch.GetId(), false)
+		if err != utils.ErrNotFound {
+			t.Error(err)
+		}
+		// */
+	}
+}

migrator/derived_chargers_test.go

@@ -0,0 +1,269 @@
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package migrator
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func TestFieldinfo2Attribute(t *testing.T) {
+	type testA struct {
+		FieldName string
+		FieldInfo string
+		Initial   []*engine.Attribute
+		Expected  []*engine.Attribute
+	}
+	tests := []testA{
+		testA{
+			FieldName: utils.Account,
+			FieldInfo: utils.MetaDefault,
+			Initial:   make([]*engine.Attribute, 0),
+			Expected:  make([]*engine.Attribute, 0),
+		},
+		testA{
+			FieldName: utils.Account,
+			FieldInfo: "",
+			Initial:   make([]*engine.Attribute, 0),
+			Expected:  make([]*engine.Attribute, 0),
+		},
+		testA{
+			FieldName: utils.Account,
+			FieldInfo: "^1003",
+			Initial:   make([]*engine.Attribute, 0),
+			Expected: []*engine.Attribute{
+				&engine.Attribute{
+					FieldName:  utils.Account,
+					Initial:    utils.META_ANY,
+					Substitute: config.NewRSRParsersMustCompile("1003", true, utils.INFIELD_SEP),
+					Append:     true,
+				},
+			},
+		},
+		testA{
+			FieldName: utils.Subject,
+			FieldInfo: `~effective_caller_id_number:s/(\d+)/+$1/`,
+			Initial:   make([]*engine.Attribute, 0),
+			Expected: []*engine.Attribute{
+				&engine.Attribute{
+					FieldName:  utils.Subject,
+					Initial:    utils.META_ANY,
+					Substitute: config.NewRSRParsersMustCompile(`~effective_caller_id_number:s/(\d+)/+$1/`, true, utils.INFIELD_SEP),
+					Append:     true,
+				},
+			},
+		},
+		testA{
+			FieldName: utils.Subject,
+			FieldInfo: "^call_1003",
+			Initial: []*engine.Attribute{
+				&engine.Attribute{
+					FieldName:  utils.Account,
+					Initial:    utils.META_ANY,
+					Substitute: config.NewRSRParsersMustCompile("1003", true, utils.INFIELD_SEP),
+					Append:     true,
+				},
+			},
+			Expected: []*engine.Attribute{
+				&engine.Attribute{
+					FieldName:  utils.Account,
+					Initial:    utils.META_ANY,
+					Substitute: config.NewRSRParsersMustCompile("1003", true, utils.INFIELD_SEP),
+					Append:     true,
+				},
+				&engine.Attribute{
+					FieldName:  utils.Subject,
+					Initial:    utils.META_ANY,
+					Substitute: config.NewRSRParsersMustCompile("call_1003", true, utils.INFIELD_SEP),
+					Append:     true,
+				},
+			},
+		},
+	}
+	for i, v := range tests {
+		if rply := fieldinfo2Attribute(v.Initial, v.FieldName, v.FieldInfo); !reflect.DeepEqual(v.Expected, rply) {
+			t.Errorf("For %v expected: %s ,recieved: %s", i, utils.ToJSON(v.Expected), utils.ToJSON(rply))
+		}
+	}
+}
+
+func TestDerivedChargers2AttributeProfile(t *testing.T) {
+	type testC struct {
+		DC       *v1DerivedCharger
+		Tenant   string
+		Key      string
+		Filters  []string
+		Expected *engine.AttributeProfile
+	}
+	tests := []testC{
+		testC{
+			DC: &v1DerivedCharger{
+				RequestTypeField: utils.MetaDefault,
+				CategoryField:    "^*voice",
+				AccountField:     "^1003",
+			},
+			Tenant:  defaultTenant,
+			Key:     "key1",
+			Filters: make([]string, 0),
+			Expected: &engine.AttributeProfile{
+				Tenant:             defaultTenant,
+				ID:                 "key1",
+				Contexts:           []string{utils.META_ANY},
+				FilterIDs:          make([]string, 0),
+				ActivationInterval: nil,
+				Attributes: []*engine.Attribute{
+					&engine.Attribute{
+						FieldName:  utils.Category,
+						Initial:    utils.META_ANY,
+						Substitute: config.NewRSRParsersMustCompile("*voice", true, utils.INFIELD_SEP),
+						Append:     true,
+					},
+					&engine.Attribute{
+						FieldName:  utils.Account,
+						Initial:    utils.META_ANY,
+						Substitute: config.NewRSRParsersMustCompile("1003", true, utils.INFIELD_SEP),
+						Append:     true,
+					},
+				},
+				Blocker: false,
+				Weight:  10,
+			},
+		},
+		testC{
+			DC: &v1DerivedCharger{
+				RequestTypeField: utils.MetaDefault,
+				CategoryField:    "^*voice",
+				AccountField:     "^1003",
+				SubjectField:     "call_1003_to_1004",
+				DestinationField: "^1004",
+			},
+			Tenant:  defaultTenant,
+			Key:     "key1",
+			Filters: []string{"*string:Subject:1005"},
+			Expected: &engine.AttributeProfile{
+				Tenant:             defaultTenant,
+				ID:                 "key1",
+				Contexts:           []string{utils.META_ANY},
+				FilterIDs:          []string{"*string:Subject:1005"},
+				ActivationInterval: nil,
+				Attributes: []*engine.Attribute{
+					&engine.Attribute{
+						FieldName:  utils.Category,
+						Initial:    utils.META_ANY,
+						Substitute: config.NewRSRParsersMustCompile("*voice", true, utils.INFIELD_SEP),
+						Append:     true,
+					},
+					&engine.Attribute{
+						FieldName:  utils.Account,
+						Initial:    utils.META_ANY,
+						Substitute: config.NewRSRParsersMustCompile("1003", true, utils.INFIELD_SEP),
+						Append:     true,
+					},
+					&engine.Attribute{
+						FieldName:  utils.Subject,
+						Initial:    utils.META_ANY,
+						Substitute: config.NewRSRParsersMustCompile("call_1003_to_1004", true, utils.INFIELD_SEP),
+						Append:     true,
+					},
+					&engine.Attribute{
+						FieldName:  utils.Destination,
+						Initial:    utils.META_ANY,
+						Substitute: config.NewRSRParsersMustCompile("1004", true, utils.INFIELD_SEP),
+						Append:     true,
+					},
+				},
+				Blocker: false,
+				Weight:  10,
+			},
+		},
+	}
+	for i, v := range tests {
+		if rply := derivedChargers2AttributeProfile(v.DC, v.Tenant, v.Key, v.Filters); !reflect.DeepEqual(v.Expected, rply) {
+			t.Errorf("For %v expected: %s ,recieved: %s", i, utils.ToJSON(v.Expected), utils.ToJSON(rply))
+		}
+	}
+}
+
+func TestDerivedChargers2Charger(t *testing.T) {
+	type testB struct {
+		DC       *v1DerivedCharger
+		Tenant   string
+		Key      string
+		Filters  []string
+		Expected *engine.ChargerProfile
+	}
+	tests := []testB{
+		testB{
+			DC: &v1DerivedCharger{
+				RunID:            "runID",
+				RunFilters:       "~Header4:s/a/${1}b/{*duration_seconds&*round:2}(b&c)",
+				RequestTypeField: utils.MetaDefault,
+				CategoryField:    "^*voice",
+				AccountField:     "^1003",
+			},
+			Tenant: defaultTenant,
+			Key:    "key2",
+			Filters: []string{
+				"*string:Category:*voice1",
+				"*string:Account:1001",
+			},
+			Expected: &engine.ChargerProfile{
+				Tenant: defaultTenant,
+				ID:     "key2",
+				FilterIDs: []string{
+					"*string:Category:*voice1",
+					"*string:Account:1001",
+					"*rsr::~Header4:s/a/${1}b/{*duration_seconds&*round:2}(b&c)",
+				},
+				ActivationInterval: nil,
+				RunID:              "runID",
+				AttributeIDs:       make([]string, 0),
+				Weight:             10,
+			},
+		},
+		testB{
+			DC: &v1DerivedCharger{
+				RunID:        "runID2",
+				RunFilters:   "^1003",
+				AccountField: "^1003",
+			},
+			Tenant:  defaultTenant,
+			Key:     "key2",
+			Filters: []string{},
+			Expected: &engine.ChargerProfile{
+				Tenant:             defaultTenant,
+				ID:                 "key2",
+				FilterIDs:          []string{"*rsr::1003"},
+				ActivationInterval: nil,
+				RunID:              "runID2",
+				AttributeIDs:       make([]string, 0),
+				Weight:             10,
+			},
+		},
+	}
+	for i, v := range tests {
+		if rply := derivedChargers2Charger(v.DC, v.Tenant, v.Key, v.Filters); !reflect.DeepEqual(v.Expected, rply) {
+			t.Errorf("For %v expected: %s ,recieved: %s", i, utils.ToJSON(v.Expected), utils.ToJSON(rply))
+		}
+	}
+}

migrator/migrator_datadb.go

@@ -52,6 +52,9 @@ type MigratorDataDB interface {
 	getV1User() (v1u *v1UserProfile, err error)
 	setV1User(us *v1UserProfile) (err error)
 	remV1User(key string) (err error)
+	getV1DerivedChargers() (v1d *v1DerivedChargersWithKey, err error)
+	setV1DerivedChargers(dc *v1DerivedChargersWithKey) (err error)
+	remV1DerivedChargers(key string) (err error)
 
 	DataManager() *engine.DataManager
 }

migrator/storage_map_datadb.go

@@ -198,3 +198,19 @@ func (v1ms *mapMigrator) setV1User(us *v1UserProfile) (err error) {
 func (v1ms *mapMigrator) remV1User(key string) (err error) {
 	return utils.ErrNotImplemented
 }
+
+// DerivedChargers methods
+//get
+func (v1ms *mapMigrator) getV1DerivedChargers() (v1d *v1DerivedChargersWithKey, err error) {
+	return nil, utils.ErrNotImplemented
+}
+
+//set
+func (v1ms *mapMigrator) setV1DerivedChargers(dc *v1DerivedChargersWithKey) (err error) {
+	return utils.ErrNotImplemented
+}
+
+//rem
+func (v1ms *mapMigrator) remV1DerivedChargers(key string) (err error) {
+	return utils.ErrNotImplemented
+}

migrator/storage_mongo_datadb.go

@@ -33,6 +33,7 @@ const (
 	v2ThresholdProfileCol  = "threshold_profiles"
 	v1AliasCol             = "aliases"
 	v1UserCol              = "users"
+	v1DerivedChargersCol   = "derived_chargers"
 )
 
 type mongoMigrator struct {
@@ -483,3 +484,41 @@ func (v1ms *mongoMigrator) remV1User(key string) (err error) {
 	_, err = v1ms.mgoDB.DB().Collection(v1UserCol).DeleteOne(v1ms.mgoDB.GetContext(), bson.M{"key": key})
 	return
 }
+
+// DerivedChargers methods
+//get
+func (v1ms *mongoMigrator) getV1DerivedChargers() (v1d *v1DerivedChargersWithKey, err error) {
+	if v1ms.cursor == nil {
+		var cursor mongo.Cursor
+		cursor, err = v1ms.mgoDB.DB().Collection(v1DerivedChargersCol).Find(v1ms.mgoDB.GetContext(), bson.D{})
+		if err != nil {
+			return nil, err
+		}
+		v1ms.cursor = &cursor
+	}
+	if !(*v1ms.cursor).Next(v1ms.mgoDB.GetContext()) {
+		(*v1ms.cursor).Close(v1ms.mgoDB.GetContext())
+		v1ms.cursor = nil
+		return nil, utils.ErrNoMoreData
+	}
+	v1d = new(v1DerivedChargersWithKey)
+	if err := (*v1ms.cursor).Decode(v1d); err != nil {
+		return nil, err
+	}
+	return v1d, nil
+}
+
+//set
+func (v1ms *mongoMigrator) setV1DerivedChargers(dc *v1DerivedChargersWithKey) (err error) {
+	_, err = v1ms.mgoDB.DB().Collection(v1DerivedChargersCol).UpdateOne(v1ms.mgoDB.GetContext(), bson.M{"key": dc.Key},
+		bson.M{"$set": dc},
+		options.Update().SetUpsert(true),
+	)
+	return
+}
+
+//rem
+func (v1ms *mongoMigrator) remV1DerivedChargers(key string) (err error) {
+	_, err = v1ms.mgoDB.DB().Collection(v1DerivedChargersCol).DeleteOne(v1ms.mgoDB.GetContext(), bson.M{"key": key})
+	return
+}

migrator/storage_redis.go

@@ -570,7 +570,7 @@ func (v1rs *redisMigrator) getV1User() (v1u *v1UserProfile, err error) {
 		if err != nil {
 			return
 		} else if len(v1rs.dataKeys) == 0 {
-			return nil, utils.ErrNotFound
+			return nil, utils.ErrNoMoreData
 		}
 		v1rs.qryIdx = utils.IntPointer(0)
 	}
@@ -603,3 +603,50 @@ func (v1rs *redisMigrator) setV1User(us *v1UserProfile) (err error) {
 func (v1rs *redisMigrator) remV1User(key string) (err error) {
 	return v1rs.rds.Cmd("DEL", utils.USERS_PREFIX+key).Err
 }
+
+// DerivedChargers methods
+//get
+func (v1rs *redisMigrator) getV1DerivedChargers() (v1d *v1DerivedChargersWithKey, err error) {
+	if v1rs.qryIdx == nil {
+		v1rs.dataKeys, err = v1rs.rds.GetKeysForPrefix(utils.DERIVEDCHARGERS_PREFIX)
+		if err != nil {
+			return
+		} else if len(v1rs.dataKeys) == 0 {
+			return nil, utils.ErrNoMoreData
+		}
+		v1rs.qryIdx = utils.IntPointer(0)
+	}
+	if *v1rs.qryIdx <= len(v1rs.dataKeys)-1 {
+		strVal, err := v1rs.rds.Cmd("GET", v1rs.dataKeys[*v1rs.qryIdx]).Bytes()
+		if err != nil {
+			return nil, err
+		}
+		v1d = new(v1DerivedChargersWithKey)
+		v1d.Key = strings.TrimPrefix(v1rs.dataKeys[*v1rs.qryIdx], utils.DERIVEDCHARGERS_PREFIX)
+		v1d.Value = new(v1DerivedChargers)
+		if err := v1rs.rds.Marshaler().Unmarshal(strVal, v1d.Value); err != nil {
+			return nil, err
+		}
+		*v1rs.qryIdx = *v1rs.qryIdx + 1
+		return v1d, nil
+	}
+	v1rs.qryIdx = nil
+	return nil, utils.ErrNoMoreData
+}
+
+//set
+func (v1rs *redisMigrator) setV1DerivedChargers(dc *v1DerivedChargersWithKey) (err error) {
+	if dc == nil || len(dc.Value.Chargers) == 0 {
+		return v1rs.remV1DerivedChargers(dc.Key)
+	}
+	bit, err := v1rs.rds.Marshaler().Marshal(dc.Value)
+	if err != nil {
+		return err
+	}
+	return v1rs.rds.Cmd("SET", utils.DERIVEDCHARGERS_PREFIX+dc.Key, bit).Err
+}
+
+//rem
+func (v1rs *redisMigrator) remV1DerivedChargers(key string) (err error) {
+	return v1rs.rds.Cmd("DEL", utils.DERIVEDCHARGERS_PREFIX+key).Err
+}

migrator/user.go

@@ -33,7 +33,6 @@ type v1UserProfile struct {
 	Masked   bool //disable if true
 	Profile  map[string]string
 	Weight   float64
-	ponder   int
 }
 
 func (ud *v1UserProfile) GetId() string {

utils/coreutils.go

@@ -340,6 +340,10 @@ func ConcatenatedKey(keyVals ...string) string {
 	return strings.Join(keyVals, CONCATENATED_KEY_SEP)
 }
 
+func SplitConcatenatedKey(key string) []string {
+	return strings.Split(key, CONCATENATED_KEY_SEP)
+}
+
 func LCRKey(direction, tenant, category, account, subject string) string {
 	return ConcatenatedKey(direction, tenant, category, account, subject)
 }