diff --git a/engine/version.go b/engine/version.go index 75ce9d231..53b2d10e2 100644 --- a/engine/version.go +++ b/engine/version.go @@ -74,6 +74,7 @@ func (vers Versions) Compare(curent Versions, storType string) string { var x map[string]string m := map[string]string{ utils.Accounts: "cgr-migrator -migrate=*accounts", + utils.Attributes: "cgr-migrator -migrate=*attributes", utils.Actions: "cgr-migrator -migrate=*actions", utils.ActionTriggers: "cgr-migrator -migrate=*action_triggers", utils.ActionPlans: "cgr-migrator -migrate=*action_plans", @@ -82,6 +83,7 @@ func (vers Versions) Compare(curent Versions, storType string) string { } data := map[string]string{ utils.Accounts: "cgr-migrator -migrate=*accounts", + utils.Attributes: "cgr-migrator -migrate=*attributes", utils.Actions: "cgr-migrator -migrate=*actions", utils.ActionTriggers: "cgr-migrator -migrate=*action_triggers", utils.ActionPlans: "cgr-migrator -migrate=*action_plans", @@ -118,6 +120,7 @@ func CurrentDataDBVersions() Versions { utils.SharedGroups: 2, utils.Thresholds: 2, utils.Suppliers: 1, + utils.Attributes: 2, utils.Timing: 1, utils.RQF: 1, utils.Resource: 1, diff --git a/migrator/attributes.go b/migrator/attributes.go new file mode 100644 index 000000000..7cacdd1a8 --- /dev/null +++ b/migrator/attributes.go @@ -0,0 +1,165 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package migrator + +import ( + "fmt" + //"log" + "strings" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +type v1Attribute struct { + FieldName string + Initial string + Substitute string + Append bool +} + +type v1AttributeProfile struct { + Tenant string + ID string + Contexts []string // bind this AttributeProfile to multiple contexts + FilterIDs []string + ActivationInterval *utils.ActivationInterval // Activation interval + Attributes map[string]map[string]*v1Attribute // map[FieldName][InitialValue]*Attribute + Weight float64 +} + +func (m *Migrator) migrateCurrentAttributeProfile() (err error) { + var ids []string + tenant := config.CgrConfig().DefaultTenant + ids, err = m.dmIN.DataDB().GetKeysForPrefix(utils.AttributeProfilePrefix) + if err != nil { + return err + } + for _, id := range ids { + idg := strings.TrimPrefix(id, utils.AttributeProfilePrefix+tenant+":") + attrPrf, err := m.dmIN.GetAttributeProfile(tenant, idg, true, utils.NonTransactional) + if err != nil { + return err + } + if attrPrf != nil { + if m.dryRun != true { + if err := m.dmOut.SetAttributeProfile(attrPrf, true); err != nil { + return err + } + } + } + } + return +} + +func (m *Migrator) migrateV1Attributes() (err error) { + var v1Attr *v1AttributeProfile + for { + v1Attr, err = m.oldDataDB.getV1AttributeProfile() + if err != nil && err != utils.ErrNoMoreData { + return err + } + if err == utils.ErrNoMoreData { + break + } + if v1Attr != nil { + attrPrf := v1Attr.AsAttributeProfile() + if err != nil { + return err + } + if m.dryRun != true { + if err := m.dmOut.DataDB().SetAttributeProfileDrv(attrPrf); err != nil { + return err + } + if err := m.dmOut.SetAttributeProfile(attrPrf, true); err != nil { + return err + } + m.stats[utils.Attributes] += 1 + } + } + } + if m.dryRun != true { + // All done, update version wtih current one + vrs := engine.Versions{utils.Attributes: engine.CurrentStorDBVersions()[utils.Attributes]} + if err = m.dmOut.DataDB().SetVersions(vrs, false); err != nil { + return utils.NewCGRError(utils.Migrator, + utils.ServerErrorCaps, + err.Error(), + fmt.Sprintf("error: <%s> when updating Thresholds version into dataDB", err.Error())) + } + } + return +} + +func (m *Migrator) migrateAttributeProfile() (err error) { + var vrs engine.Versions + current := engine.CurrentDataDBVersions() + vrs, err = m.dmOut.DataDB().GetVersions(utils.TBLVersions) + if err != nil { + return utils.NewCGRError(utils.Migrator, + utils.ServerErrorCaps, + err.Error(), + fmt.Sprintf("error: <%s> when querying oldDataDB for versions", err.Error())) + } else if len(vrs) == 0 { + return utils.NewCGRError(utils.Migrator, + utils.MandatoryIEMissingCaps, + utils.UndefinedVersion, + "version number is not defined for ActionTriggers model") + } + switch vrs[utils.Attributes] { + case current[utils.Attributes]: + if m.sameDataDB { + return + } + if err := m.migrateCurrentAttributeProfile(); err != nil { + return err + } + return + case 1: + if err := m.migrateV1Attributes(); err != nil { + return err + } + } + return +} + +func (v1AttrPrf v1AttributeProfile) AsAttributeProfile() (attrPrf *engine.AttributeProfile) { + attrPrf = &engine.AttributeProfile{ + Tenant: v1AttrPrf.Tenant, + ID: v1AttrPrf.ID, + Contexts: v1AttrPrf.Contexts, + FilterIDs: v1AttrPrf.FilterIDs, + Weight: v1AttrPrf.Weight, + ActivationInterval: v1AttrPrf.ActivationInterval, + } + for _, mp := range v1AttrPrf.Attributes { + for _, attr := range mp { + initIface := utils.StringToInterface(attr.Initial) + substituteIface := utils.StringToInterface(attr.Substitute) + attrPrf.Attributes = append(attrPrf.Attributes, &engine.Attribute{ + FieldName: attr.FieldName, + Initial: initIface, + Substitute: substituteIface, + Append: attr.Append, + }) + } + } + return +} diff --git a/migrator/attributes_test.go b/migrator/attributes_test.go new file mode 100644 index 000000000..a6d415411 --- /dev/null +++ b/migrator/attributes_test.go @@ -0,0 +1,77 @@ +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ +package migrator + +import ( + "reflect" + "testing" + "time" + + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +func Testv1AttributeProfileAsAttributeProfile(t *testing.T) { + var cloneExpTime time.Time + expTime := time.Now().Add(time.Duration(20 * time.Minute)) + if err := utils.Clone(expTime, &cloneExpTime); err != nil { + t.Error(err) + } + mapSubstitutes := make(map[string]map[string]*v1Attribute) + mapSubstitutes["FL1"] = make(map[string]*v1Attribute) + mapSubstitutes["FL1"]["In1"] = &v1Attribute{ + FieldName: "FL1", + Initial: "In1", + Substitute: "Al1", + Append: true, + } + v1Attribute := &v1AttributeProfile{ + Tenant: "cgrates.org", + ID: "attributeprofile1", + Contexts: []string{utils.MetaRating}, + FilterIDs: []string{"filter1"}, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + ExpiryTime: cloneExpTime, + }, + Attributes: mapSubstitutes, + Weight: 20, + } + attrPrf := &engine.AttributeProfile{ + Tenant: "cgrates.org", + ID: "attributeprofile1", + Contexts: []string{utils.MetaRating}, + FilterIDs: []string{"filter1"}, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + ExpiryTime: cloneExpTime, + }, + Attributes: []*engine.Attribute{ + &engine.Attribute{ + FieldName: "FL1", + Initial: "In1", + Substitute: "Al1", + Append: true, + }, + }, + Weight: 20, + } + if !reflect.DeepEqual(attrPrf, v1Attribute.AsAttributeProfile()) { + t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(attrPrf), utils.ToJSON(v1Attribute.AsAttributeProfile())) + } +} diff --git a/migrator/migrator.go b/migrator/migrator.go index 0062cb17f..42c941a6a 100755 --- a/migrator/migrator.go +++ b/migrator/migrator.go @@ -131,6 +131,8 @@ func (m *Migrator) Migrate(taskIDs []string) (err error, stats map[string]int) { err = m.migrateStats() case utils.MetaThresholds: err = m.migrateThresholds() + case utils.MetaAttributes: + err = m.migrateAttributeProfile() //only Move case utils.MetaRatingPlans: err = m.migrateRatingPlans() @@ -229,6 +231,9 @@ func (m *Migrator) Migrate(taskIDs []string) (err error, stats map[string]int) { if err := m.migrateSupplierProfiles(); err != nil { log.Print("ERROR: ", utils.MetaSuppliers, " ", err) } + if err := m.migrateAttributeProfile(); err != nil { + log.Print("ERROR: ", utils.MetaAttributes, " ", err) + } if err := m.migrateRatingPlans(); err != nil { log.Print("ERROR: ", utils.MetaRatingPlans, " ", err) } diff --git a/migrator/migratorDataDB.go b/migrator/migratorDataDB.go index f7f1034ff..c46f745f9 100644 --- a/migrator/migratorDataDB.go +++ b/migrator/migratorDataDB.go @@ -36,4 +36,6 @@ type MigratorDataDB interface { setV2ActionTrigger(x *v2ActionTrigger) (err error) getv2Account() (v2Acnt *v2Account, err error) setV2Account(x *v2Account) (err error) + getV1AttributeProfile() (v1attrPrf *v1AttributeProfile, err error) + setV1AttributeProfile(x *v1AttributeProfile) (err error) } diff --git a/migrator/migrator_it_test.go b/migrator/migrator_it_test.go index 58cec6a63..49ce5b6a4 100644 --- a/migrator/migrator_it_test.go +++ b/migrator/migrator_it_test.go @@ -65,6 +65,7 @@ var sTestsITMigrator = []func(t *testing.T){ testMigratorSubscribers, testMigratorTimings, testMigratorThreshold, + testMigratorAttributeProfile, //TPS testMigratorTPRatingProfile, testMigratorTPSuppliers, @@ -1781,6 +1782,164 @@ func testMigratorTimings(t *testing.T) { } } +func testMigratorAttributeProfile(t *testing.T) { + mapSubstitutes := make(map[string]map[string]*v1Attribute) + mapSubstitutes["FL1"] = make(map[string]*v1Attribute) + mapSubstitutes["FL1"]["In1"] = &v1Attribute{ + FieldName: "FL1", + Initial: "In1", + Substitute: "Al1", + Append: true, + } + v1Attribute := &v1AttributeProfile{ + Tenant: "cgrates.org", + ID: "attributeprofile1", + Contexts: []string{utils.MetaRating}, + FilterIDs: []string{"filter1"}, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + ExpiryTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + }, + Attributes: mapSubstitutes, + Weight: 20, + } + attrPrf := &engine.AttributeProfile{ + Tenant: "cgrates.org", + ID: "attributeprofile1", + Contexts: []string{utils.MetaRating}, + FilterIDs: []string{"filter1"}, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + ExpiryTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + }, + Attributes: []*engine.Attribute{ + &engine.Attribute{ + FieldName: "FL1", + Initial: "In1", + Substitute: "Al1", + Append: true, + }, + }, + Weight: 20, + } + filterAttr := &engine.Filter{ + Tenant: attrPrf.Tenant, + ID: attrPrf.FilterIDs[0], + RequestFilters: []*engine.RequestFilter{ + &engine.RequestFilter{ + FieldName: "Name", + Type: "Type", + Values: []string{"Val1"}, + }, + }, + } + switch { + case action == utils.REDIS: + if err := mig.dmIN.SetFilter(filterAttr); err != nil { + t.Error("Error when setting Filter ", err.Error()) + } + if err := mig.dmIN.SetAttributeProfile(attrPrf, true); err != nil { + t.Error("Error when setting attributeProfile ", err.Error()) + } + err := mig.oldDataDB.setV1AttributeProfile(v1Attribute) + if err != nil { + t.Error("Error when setting V1AttributeProfile ", err.Error()) + } + currentVersion := engine.CurrentDataDBVersions() + currentVersion[utils.Attributes] = 1 + err = mig.dmOut.DataDB().SetVersions(currentVersion, false) + if err != nil { + t.Error("Error when setting version for attributeProfile ", err.Error()) + } + err, _ = mig.Migrate([]string{utils.MetaAttributes}) + if err != nil { + t.Error("Error when migrating AttributeProfile ", err.Error()) + } + result, err := mig.dmOut.GetAttributeProfile(attrPrf.Tenant, attrPrf.ID, true, utils.NonTransactional) + if err != nil { + t.Error("Error when getting AttributeProfile ", err.Error()) + } + if !reflect.DeepEqual(attrPrf.Tenant, result.Tenant) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.Tenant, result.Tenant) + } else if !reflect.DeepEqual(attrPrf.ID, result.ID) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.ID, result.ID) + } else if !reflect.DeepEqual(attrPrf.FilterIDs, result.FilterIDs) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.FilterIDs, result.FilterIDs) + } else if !reflect.DeepEqual(attrPrf.Contexts, result.Contexts) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.Contexts, result.Contexts) + } else if !reflect.DeepEqual(attrPrf.ActivationInterval, result.ActivationInterval) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.ActivationInterval, result.ActivationInterval) + } else if !reflect.DeepEqual(attrPrf.Attributes, result.Attributes) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.Attributes, result.Attributes) + } + case action == utils.MONGO: + if err := mig.dmIN.SetAttributeProfile(attrPrf, true); err != nil { + t.Error("Error when setting attributeProfile ", err.Error()) + } + err := mig.oldDataDB.setV1AttributeProfile(v1Attribute) + if err != nil { + t.Error("Error when setting V1AttributeProfile ", err.Error()) + } + currentVersion := engine.CurrentDataDBVersions() + currentVersion[utils.Attributes] = 1 + err = mig.dmOut.DataDB().SetVersions(currentVersion, false) + if err != nil { + t.Error("Error when setting version for attributeProfile ", err.Error()) + } + err, _ = mig.Migrate([]string{utils.MetaAttributes}) + if err != nil { + t.Error("Error when migrating attributeProfile ", err.Error()) + } + result, err := mig.dmOut.GetAttributeProfile("cgrates.org", attrPrf.ID, true, utils.NonTransactional) + if err != nil { + t.Error("Error when getting attributeProfile ", err.Error()) + } + if !reflect.DeepEqual(attrPrf.Tenant, result.Tenant) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.Tenant, result.Tenant) + } else if !reflect.DeepEqual(attrPrf.ID, result.ID) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.ID, result.ID) + } else if !reflect.DeepEqual(attrPrf.FilterIDs, result.FilterIDs) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.FilterIDs, result.FilterIDs) + } else if !reflect.DeepEqual(attrPrf.Contexts, result.Contexts) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.Contexts, result.Contexts) + } else if !reflect.DeepEqual(attrPrf.ActivationInterval, result.ActivationInterval) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.ActivationInterval, result.ActivationInterval) + } else if !reflect.DeepEqual(attrPrf.Attributes, result.Attributes) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.Attributes, result.Attributes) + } + case action == Move: + if err := mig.dmIN.SetAttributeProfile(attrPrf, true); err != nil { + t.Error("Error when setting AttributeProfile ", err.Error()) + } + currentVersion := engine.CurrentDataDBVersions() + err := mig.dmOut.DataDB().SetVersions(currentVersion, false) + if err != nil { + t.Error("Error when setting version for stats ", err.Error()) + } + err, _ = mig.Migrate([]string{utils.MetaAttributes}) + if err != nil { + t.Error("Error when migrating AttributeProfile ", err.Error()) + } + result, err := mig.dmOut.GetAttributeProfile(attrPrf.Tenant, attrPrf.ID, true, utils.NonTransactional) + if err != nil { + t.Error("Error when getting Stats ", err.Error()) + } + if !reflect.DeepEqual(attrPrf.Tenant, result.Tenant) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.Tenant, result.Tenant) + } else if !reflect.DeepEqual(attrPrf.ID, result.ID) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.ID, result.ID) + } else if !reflect.DeepEqual(attrPrf.FilterIDs, result.FilterIDs) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.FilterIDs, result.FilterIDs) + } else if !reflect.DeepEqual(attrPrf.Contexts, result.Contexts) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.Contexts, result.Contexts) + } else if !reflect.DeepEqual(attrPrf.ActivationInterval, result.ActivationInterval) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.ActivationInterval, result.ActivationInterval) + } else if !reflect.DeepEqual(attrPrf.Attributes, result.Attributes) { + t.Errorf("Expecting: %+v, received: %+v", attrPrf.Attributes, result.Attributes) + } + } +} + //TP TESTS func testMigratorTPRatingProfile(t *testing.T) { tpRatingProfile := []*utils.TPRatingProfile{ diff --git a/migrator/thresholds.go b/migrator/thresholds.go index 8f2b52929..1a4c94444 100644 --- a/migrator/thresholds.go +++ b/migrator/thresholds.go @@ -50,7 +50,7 @@ type v2ActionTriggers []*v2ActionTrigger func (m *Migrator) migrateCurrentThresholds() (err error) { var ids []string tenant := config.CgrConfig().DefaultTenant - //StatQueue + //Thresholds ids, err = m.dmIN.DataDB().GetKeysForPrefix(utils.ThresholdPrefix) if err != nil { return err @@ -70,7 +70,7 @@ func (m *Migrator) migrateCurrentThresholds() (err error) { } } } - //StatQueueProfile + //ThresholdProfiles ids, err = m.dmIN.DataDB().GetKeysForPrefix(utils.ThresholdProfilePrefix) if err != nil { return err @@ -166,6 +166,7 @@ func (m *Migrator) migrateThresholds() (err error) { } return } + func (v2ATR v2ActionTrigger) AsThreshold() (thp *engine.ThresholdProfile, th *engine.Threshold, filter *engine.Filter, err error) { var filterIDS []string var filters []*engine.RequestFilter diff --git a/migrator/v1mongo_data.go b/migrator/v1mongo_data.go index b8a6f2d9a..3e66df38d 100644 --- a/migrator/v1mongo_data.go +++ b/migrator/v1mongo_data.go @@ -245,3 +245,26 @@ func (v1ms *v1Mongo) setV2ActionTrigger(x *v2ActionTrigger) (err error) { } return } + +//AttributeProfile methods +//get +func (v1ms *v1Mongo) getV1AttributeProfile() (v1attrPrf *v1AttributeProfile, err error) { + if v1ms.qryIter == nil { + v1ms.qryIter = v1ms.session.DB(v1ms.db).C(utils.AttributeProfilePrefix).Find(nil).Iter() + } + v1ms.qryIter.Next(&v1attrPrf) + if v1attrPrf == nil { + v1ms.qryIter = nil + return nil, utils.ErrNoMoreData + + } + return v1attrPrf, nil +} + +//set +func (v1ms *v1Mongo) setV1AttributeProfile(x *v1AttributeProfile) (err error) { + if err := v1ms.session.DB(v1ms.db).C(utils.AttributeProfilePrefix).Insert(x); err != nil { + return err + } + return +} diff --git a/migrator/v1redis.go b/migrator/v1redis.go index ec01edcb0..ce1757a90 100644 --- a/migrator/v1redis.go +++ b/migrator/v1redis.go @@ -459,3 +459,45 @@ func (v1rs *v1Redis) setV2ActionTrigger(x *v2ActionTrigger) (err error) { } return } + +//AttributeProfile methods +//get +func (v1rs *v1Redis) getV1AttributeProfile() (v1attrPrf *v1AttributeProfile, err error) { + var v1attr *v1AttributeProfile + if v1rs.qryIdx == nil { + v1rs.dataKeys, err = v1rs.getKeysForPrefix(utils.AttributeProfilePrefix) + if err != nil { + return + } else if len(v1rs.dataKeys) == 0 { + return nil, utils.ErrNotFound + } + v1rs.qryIdx = utils.IntPointer(0) + } + if *v1rs.qryIdx <= len(v1rs.dataKeys)-1 { + strVal, err := v1rs.cmd("GET", v1rs.dataKeys[*v1rs.qryIdx]).Bytes() + if err != nil { + return nil, err + } + if err := v1rs.ms.Unmarshal(strVal, &v1attr); err != nil { + return nil, err + } + *v1rs.qryIdx = *v1rs.qryIdx + 1 + } else { + v1rs.qryIdx = nil + return nil, utils.ErrNoMoreData + } + return v1attr, nil +} + +//set +func (v1rs *v1Redis) setV1AttributeProfile(x *v1AttributeProfile) (err error) { + key := utils.AttributeProfilePrefix + utils.ConcatenatedKey(x.Tenant, x.ID) + bit, err := v1rs.ms.Marshal(x) + if err != nil { + return err + } + if err = v1rs.cmd("SET", key, bit).Err; err != nil { + return err + } + return +} diff --git a/utils/consts.go b/utils/consts.go index 5c749fb22..e03c65932 100755 --- a/utils/consts.go +++ b/utils/consts.go @@ -407,6 +407,7 @@ const ( ID = "ID" Thresholds = "Thresholds" Suppliers = "Suppliers" + Attributes = "Attributes" StatS = "stats" RALService = "RALs" CostSource = "CostSource"