Add AttributeProfiles to migrator

This commit is contained in:
TeoV
2018-01-26 19:21:48 +02:00
committed by Dan Christian Bogos
parent 879673d638
commit 16ca77bad9
10 changed files with 480 additions and 2 deletions

View File

@@ -74,6 +74,7 @@ func (vers Versions) Compare(curent Versions, storType string) string {
var x map[string]string
m := map[string]string{
utils.Accounts: "cgr-migrator -migrate=*accounts",
utils.Attributes: "cgr-migrator -migrate=*attributes",
utils.Actions: "cgr-migrator -migrate=*actions",
utils.ActionTriggers: "cgr-migrator -migrate=*action_triggers",
utils.ActionPlans: "cgr-migrator -migrate=*action_plans",
@@ -82,6 +83,7 @@ func (vers Versions) Compare(curent Versions, storType string) string {
}
data := map[string]string{
utils.Accounts: "cgr-migrator -migrate=*accounts",
utils.Attributes: "cgr-migrator -migrate=*attributes",
utils.Actions: "cgr-migrator -migrate=*actions",
utils.ActionTriggers: "cgr-migrator -migrate=*action_triggers",
utils.ActionPlans: "cgr-migrator -migrate=*action_plans",
@@ -118,6 +120,7 @@ func CurrentDataDBVersions() Versions {
utils.SharedGroups: 2,
utils.Thresholds: 2,
utils.Suppliers: 1,
utils.Attributes: 2,
utils.Timing: 1,
utils.RQF: 1,
utils.Resource: 1,

165
migrator/attributes.go Normal file
View File

@@ -0,0 +1,165 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package migrator
import (
"fmt"
//"log"
"strings"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
type v1Attribute struct {
FieldName string
Initial string
Substitute string
Append bool
}
type v1AttributeProfile struct {
Tenant string
ID string
Contexts []string // bind this AttributeProfile to multiple contexts
FilterIDs []string
ActivationInterval *utils.ActivationInterval // Activation interval
Attributes map[string]map[string]*v1Attribute // map[FieldName][InitialValue]*Attribute
Weight float64
}
func (m *Migrator) migrateCurrentAttributeProfile() (err error) {
var ids []string
tenant := config.CgrConfig().DefaultTenant
ids, err = m.dmIN.DataDB().GetKeysForPrefix(utils.AttributeProfilePrefix)
if err != nil {
return err
}
for _, id := range ids {
idg := strings.TrimPrefix(id, utils.AttributeProfilePrefix+tenant+":")
attrPrf, err := m.dmIN.GetAttributeProfile(tenant, idg, true, utils.NonTransactional)
if err != nil {
return err
}
if attrPrf != nil {
if m.dryRun != true {
if err := m.dmOut.SetAttributeProfile(attrPrf, true); err != nil {
return err
}
}
}
}
return
}
func (m *Migrator) migrateV1Attributes() (err error) {
var v1Attr *v1AttributeProfile
for {
v1Attr, err = m.oldDataDB.getV1AttributeProfile()
if err != nil && err != utils.ErrNoMoreData {
return err
}
if err == utils.ErrNoMoreData {
break
}
if v1Attr != nil {
attrPrf := v1Attr.AsAttributeProfile()
if err != nil {
return err
}
if m.dryRun != true {
if err := m.dmOut.DataDB().SetAttributeProfileDrv(attrPrf); err != nil {
return err
}
if err := m.dmOut.SetAttributeProfile(attrPrf, true); err != nil {
return err
}
m.stats[utils.Attributes] += 1
}
}
}
if m.dryRun != true {
// All done, update version wtih current one
vrs := engine.Versions{utils.Attributes: engine.CurrentStorDBVersions()[utils.Attributes]}
if err = m.dmOut.DataDB().SetVersions(vrs, false); err != nil {
return utils.NewCGRError(utils.Migrator,
utils.ServerErrorCaps,
err.Error(),
fmt.Sprintf("error: <%s> when updating Thresholds version into dataDB", err.Error()))
}
}
return
}
func (m *Migrator) migrateAttributeProfile() (err error) {
var vrs engine.Versions
current := engine.CurrentDataDBVersions()
vrs, err = m.dmOut.DataDB().GetVersions(utils.TBLVersions)
if err != nil {
return utils.NewCGRError(utils.Migrator,
utils.ServerErrorCaps,
err.Error(),
fmt.Sprintf("error: <%s> when querying oldDataDB for versions", err.Error()))
} else if len(vrs) == 0 {
return utils.NewCGRError(utils.Migrator,
utils.MandatoryIEMissingCaps,
utils.UndefinedVersion,
"version number is not defined for ActionTriggers model")
}
switch vrs[utils.Attributes] {
case current[utils.Attributes]:
if m.sameDataDB {
return
}
if err := m.migrateCurrentAttributeProfile(); err != nil {
return err
}
return
case 1:
if err := m.migrateV1Attributes(); err != nil {
return err
}
}
return
}
func (v1AttrPrf v1AttributeProfile) AsAttributeProfile() (attrPrf *engine.AttributeProfile) {
attrPrf = &engine.AttributeProfile{
Tenant: v1AttrPrf.Tenant,
ID: v1AttrPrf.ID,
Contexts: v1AttrPrf.Contexts,
FilterIDs: v1AttrPrf.FilterIDs,
Weight: v1AttrPrf.Weight,
ActivationInterval: v1AttrPrf.ActivationInterval,
}
for _, mp := range v1AttrPrf.Attributes {
for _, attr := range mp {
initIface := utils.StringToInterface(attr.Initial)
substituteIface := utils.StringToInterface(attr.Substitute)
attrPrf.Attributes = append(attrPrf.Attributes, &engine.Attribute{
FieldName: attr.FieldName,
Initial: initIface,
Substitute: substituteIface,
Append: attr.Append,
})
}
}
return
}

View File

@@ -0,0 +1,77 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package migrator
import (
"reflect"
"testing"
"time"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
func Testv1AttributeProfileAsAttributeProfile(t *testing.T) {
var cloneExpTime time.Time
expTime := time.Now().Add(time.Duration(20 * time.Minute))
if err := utils.Clone(expTime, &cloneExpTime); err != nil {
t.Error(err)
}
mapSubstitutes := make(map[string]map[string]*v1Attribute)
mapSubstitutes["FL1"] = make(map[string]*v1Attribute)
mapSubstitutes["FL1"]["In1"] = &v1Attribute{
FieldName: "FL1",
Initial: "In1",
Substitute: "Al1",
Append: true,
}
v1Attribute := &v1AttributeProfile{
Tenant: "cgrates.org",
ID: "attributeprofile1",
Contexts: []string{utils.MetaRating},
FilterIDs: []string{"filter1"},
ActivationInterval: &utils.ActivationInterval{
ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC),
ExpiryTime: cloneExpTime,
},
Attributes: mapSubstitutes,
Weight: 20,
}
attrPrf := &engine.AttributeProfile{
Tenant: "cgrates.org",
ID: "attributeprofile1",
Contexts: []string{utils.MetaRating},
FilterIDs: []string{"filter1"},
ActivationInterval: &utils.ActivationInterval{
ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC),
ExpiryTime: cloneExpTime,
},
Attributes: []*engine.Attribute{
&engine.Attribute{
FieldName: "FL1",
Initial: "In1",
Substitute: "Al1",
Append: true,
},
},
Weight: 20,
}
if !reflect.DeepEqual(attrPrf, v1Attribute.AsAttributeProfile()) {
t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(attrPrf), utils.ToJSON(v1Attribute.AsAttributeProfile()))
}
}

View File

@@ -131,6 +131,8 @@ func (m *Migrator) Migrate(taskIDs []string) (err error, stats map[string]int) {
err = m.migrateStats()
case utils.MetaThresholds:
err = m.migrateThresholds()
case utils.MetaAttributes:
err = m.migrateAttributeProfile()
//only Move
case utils.MetaRatingPlans:
err = m.migrateRatingPlans()
@@ -229,6 +231,9 @@ func (m *Migrator) Migrate(taskIDs []string) (err error, stats map[string]int) {
if err := m.migrateSupplierProfiles(); err != nil {
log.Print("ERROR: ", utils.MetaSuppliers, " ", err)
}
if err := m.migrateAttributeProfile(); err != nil {
log.Print("ERROR: ", utils.MetaAttributes, " ", err)
}
if err := m.migrateRatingPlans(); err != nil {
log.Print("ERROR: ", utils.MetaRatingPlans, " ", err)
}

View File

@@ -36,4 +36,6 @@ type MigratorDataDB interface {
setV2ActionTrigger(x *v2ActionTrigger) (err error)
getv2Account() (v2Acnt *v2Account, err error)
setV2Account(x *v2Account) (err error)
getV1AttributeProfile() (v1attrPrf *v1AttributeProfile, err error)
setV1AttributeProfile(x *v1AttributeProfile) (err error)
}

View File

@@ -65,6 +65,7 @@ var sTestsITMigrator = []func(t *testing.T){
testMigratorSubscribers,
testMigratorTimings,
testMigratorThreshold,
testMigratorAttributeProfile,
//TPS
testMigratorTPRatingProfile,
testMigratorTPSuppliers,
@@ -1781,6 +1782,164 @@ func testMigratorTimings(t *testing.T) {
}
}
func testMigratorAttributeProfile(t *testing.T) {
mapSubstitutes := make(map[string]map[string]*v1Attribute)
mapSubstitutes["FL1"] = make(map[string]*v1Attribute)
mapSubstitutes["FL1"]["In1"] = &v1Attribute{
FieldName: "FL1",
Initial: "In1",
Substitute: "Al1",
Append: true,
}
v1Attribute := &v1AttributeProfile{
Tenant: "cgrates.org",
ID: "attributeprofile1",
Contexts: []string{utils.MetaRating},
FilterIDs: []string{"filter1"},
ActivationInterval: &utils.ActivationInterval{
ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC),
ExpiryTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC),
},
Attributes: mapSubstitutes,
Weight: 20,
}
attrPrf := &engine.AttributeProfile{
Tenant: "cgrates.org",
ID: "attributeprofile1",
Contexts: []string{utils.MetaRating},
FilterIDs: []string{"filter1"},
ActivationInterval: &utils.ActivationInterval{
ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC),
ExpiryTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC),
},
Attributes: []*engine.Attribute{
&engine.Attribute{
FieldName: "FL1",
Initial: "In1",
Substitute: "Al1",
Append: true,
},
},
Weight: 20,
}
filterAttr := &engine.Filter{
Tenant: attrPrf.Tenant,
ID: attrPrf.FilterIDs[0],
RequestFilters: []*engine.RequestFilter{
&engine.RequestFilter{
FieldName: "Name",
Type: "Type",
Values: []string{"Val1"},
},
},
}
switch {
case action == utils.REDIS:
if err := mig.dmIN.SetFilter(filterAttr); err != nil {
t.Error("Error when setting Filter ", err.Error())
}
if err := mig.dmIN.SetAttributeProfile(attrPrf, true); err != nil {
t.Error("Error when setting attributeProfile ", err.Error())
}
err := mig.oldDataDB.setV1AttributeProfile(v1Attribute)
if err != nil {
t.Error("Error when setting V1AttributeProfile ", err.Error())
}
currentVersion := engine.CurrentDataDBVersions()
currentVersion[utils.Attributes] = 1
err = mig.dmOut.DataDB().SetVersions(currentVersion, false)
if err != nil {
t.Error("Error when setting version for attributeProfile ", err.Error())
}
err, _ = mig.Migrate([]string{utils.MetaAttributes})
if err != nil {
t.Error("Error when migrating AttributeProfile ", err.Error())
}
result, err := mig.dmOut.GetAttributeProfile(attrPrf.Tenant, attrPrf.ID, true, utils.NonTransactional)
if err != nil {
t.Error("Error when getting AttributeProfile ", err.Error())
}
if !reflect.DeepEqual(attrPrf.Tenant, result.Tenant) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.Tenant, result.Tenant)
} else if !reflect.DeepEqual(attrPrf.ID, result.ID) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.ID, result.ID)
} else if !reflect.DeepEqual(attrPrf.FilterIDs, result.FilterIDs) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.FilterIDs, result.FilterIDs)
} else if !reflect.DeepEqual(attrPrf.Contexts, result.Contexts) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.Contexts, result.Contexts)
} else if !reflect.DeepEqual(attrPrf.ActivationInterval, result.ActivationInterval) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.ActivationInterval, result.ActivationInterval)
} else if !reflect.DeepEqual(attrPrf.Attributes, result.Attributes) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.Attributes, result.Attributes)
}
case action == utils.MONGO:
if err := mig.dmIN.SetAttributeProfile(attrPrf, true); err != nil {
t.Error("Error when setting attributeProfile ", err.Error())
}
err := mig.oldDataDB.setV1AttributeProfile(v1Attribute)
if err != nil {
t.Error("Error when setting V1AttributeProfile ", err.Error())
}
currentVersion := engine.CurrentDataDBVersions()
currentVersion[utils.Attributes] = 1
err = mig.dmOut.DataDB().SetVersions(currentVersion, false)
if err != nil {
t.Error("Error when setting version for attributeProfile ", err.Error())
}
err, _ = mig.Migrate([]string{utils.MetaAttributes})
if err != nil {
t.Error("Error when migrating attributeProfile ", err.Error())
}
result, err := mig.dmOut.GetAttributeProfile("cgrates.org", attrPrf.ID, true, utils.NonTransactional)
if err != nil {
t.Error("Error when getting attributeProfile ", err.Error())
}
if !reflect.DeepEqual(attrPrf.Tenant, result.Tenant) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.Tenant, result.Tenant)
} else if !reflect.DeepEqual(attrPrf.ID, result.ID) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.ID, result.ID)
} else if !reflect.DeepEqual(attrPrf.FilterIDs, result.FilterIDs) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.FilterIDs, result.FilterIDs)
} else if !reflect.DeepEqual(attrPrf.Contexts, result.Contexts) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.Contexts, result.Contexts)
} else if !reflect.DeepEqual(attrPrf.ActivationInterval, result.ActivationInterval) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.ActivationInterval, result.ActivationInterval)
} else if !reflect.DeepEqual(attrPrf.Attributes, result.Attributes) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.Attributes, result.Attributes)
}
case action == Move:
if err := mig.dmIN.SetAttributeProfile(attrPrf, true); err != nil {
t.Error("Error when setting AttributeProfile ", err.Error())
}
currentVersion := engine.CurrentDataDBVersions()
err := mig.dmOut.DataDB().SetVersions(currentVersion, false)
if err != nil {
t.Error("Error when setting version for stats ", err.Error())
}
err, _ = mig.Migrate([]string{utils.MetaAttributes})
if err != nil {
t.Error("Error when migrating AttributeProfile ", err.Error())
}
result, err := mig.dmOut.GetAttributeProfile(attrPrf.Tenant, attrPrf.ID, true, utils.NonTransactional)
if err != nil {
t.Error("Error when getting Stats ", err.Error())
}
if !reflect.DeepEqual(attrPrf.Tenant, result.Tenant) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.Tenant, result.Tenant)
} else if !reflect.DeepEqual(attrPrf.ID, result.ID) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.ID, result.ID)
} else if !reflect.DeepEqual(attrPrf.FilterIDs, result.FilterIDs) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.FilterIDs, result.FilterIDs)
} else if !reflect.DeepEqual(attrPrf.Contexts, result.Contexts) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.Contexts, result.Contexts)
} else if !reflect.DeepEqual(attrPrf.ActivationInterval, result.ActivationInterval) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.ActivationInterval, result.ActivationInterval)
} else if !reflect.DeepEqual(attrPrf.Attributes, result.Attributes) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf.Attributes, result.Attributes)
}
}
}
//TP TESTS
func testMigratorTPRatingProfile(t *testing.T) {
tpRatingProfile := []*utils.TPRatingProfile{

View File

@@ -50,7 +50,7 @@ type v2ActionTriggers []*v2ActionTrigger
func (m *Migrator) migrateCurrentThresholds() (err error) {
var ids []string
tenant := config.CgrConfig().DefaultTenant
//StatQueue
//Thresholds
ids, err = m.dmIN.DataDB().GetKeysForPrefix(utils.ThresholdPrefix)
if err != nil {
return err
@@ -70,7 +70,7 @@ func (m *Migrator) migrateCurrentThresholds() (err error) {
}
}
}
//StatQueueProfile
//ThresholdProfiles
ids, err = m.dmIN.DataDB().GetKeysForPrefix(utils.ThresholdProfilePrefix)
if err != nil {
return err
@@ -166,6 +166,7 @@ func (m *Migrator) migrateThresholds() (err error) {
}
return
}
func (v2ATR v2ActionTrigger) AsThreshold() (thp *engine.ThresholdProfile, th *engine.Threshold, filter *engine.Filter, err error) {
var filterIDS []string
var filters []*engine.RequestFilter

View File

@@ -245,3 +245,26 @@ func (v1ms *v1Mongo) setV2ActionTrigger(x *v2ActionTrigger) (err error) {
}
return
}
//AttributeProfile methods
//get
func (v1ms *v1Mongo) getV1AttributeProfile() (v1attrPrf *v1AttributeProfile, err error) {
if v1ms.qryIter == nil {
v1ms.qryIter = v1ms.session.DB(v1ms.db).C(utils.AttributeProfilePrefix).Find(nil).Iter()
}
v1ms.qryIter.Next(&v1attrPrf)
if v1attrPrf == nil {
v1ms.qryIter = nil
return nil, utils.ErrNoMoreData
}
return v1attrPrf, nil
}
//set
func (v1ms *v1Mongo) setV1AttributeProfile(x *v1AttributeProfile) (err error) {
if err := v1ms.session.DB(v1ms.db).C(utils.AttributeProfilePrefix).Insert(x); err != nil {
return err
}
return
}

View File

@@ -459,3 +459,45 @@ func (v1rs *v1Redis) setV2ActionTrigger(x *v2ActionTrigger) (err error) {
}
return
}
//AttributeProfile methods
//get
func (v1rs *v1Redis) getV1AttributeProfile() (v1attrPrf *v1AttributeProfile, err error) {
var v1attr *v1AttributeProfile
if v1rs.qryIdx == nil {
v1rs.dataKeys, err = v1rs.getKeysForPrefix(utils.AttributeProfilePrefix)
if err != nil {
return
} else if len(v1rs.dataKeys) == 0 {
return nil, utils.ErrNotFound
}
v1rs.qryIdx = utils.IntPointer(0)
}
if *v1rs.qryIdx <= len(v1rs.dataKeys)-1 {
strVal, err := v1rs.cmd("GET", v1rs.dataKeys[*v1rs.qryIdx]).Bytes()
if err != nil {
return nil, err
}
if err := v1rs.ms.Unmarshal(strVal, &v1attr); err != nil {
return nil, err
}
*v1rs.qryIdx = *v1rs.qryIdx + 1
} else {
v1rs.qryIdx = nil
return nil, utils.ErrNoMoreData
}
return v1attr, nil
}
//set
func (v1rs *v1Redis) setV1AttributeProfile(x *v1AttributeProfile) (err error) {
key := utils.AttributeProfilePrefix + utils.ConcatenatedKey(x.Tenant, x.ID)
bit, err := v1rs.ms.Marshal(x)
if err != nil {
return err
}
if err = v1rs.cmd("SET", key, bit).Err; err != nil {
return err
}
return
}

View File

@@ -407,6 +407,7 @@ const (
ID = "ID"
Thresholds = "Thresholds"
Suppliers = "Suppliers"
Attributes = "Attributes"
StatS = "stats"
RALService = "RALs"
CostSource = "CostSource"