uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-11 18:16:24 +05:00
Code Issues Packages Projects Releases Wiki Activity

Update migrator

Browse Source
This commit is contained in:
TeoV
2018-07-25 09:48:56 -04:00
committed by Dan Christian Bogos
parent fd3aa24b48
commit 3edc941adf
6 changed files with 147 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
engine/model_helpers.go
View File

@@ -1944,17 +1944,19 @@ func APItoResource(tpRL *utils.TPResource, timezone string) (rp *ResourceProfile
Blocker: tpRL.Blocker,
Stored: tpRL.Stored,
AllocationMessage: tpRL.AllocationMessage,
ThresholdIDs: make([]string, len(tpRL.ThresholdIDs)),
FilterIDs: make([]string, len(tpRL.FilterIDs)),
}
if tpRL.UsageTTL != "" {
if rp.UsageTTL, err = utils.ParseDurationWithNanosecs(tpRL.UsageTTL); err != nil {
return nil, err
}
}
for _, fltr := range tpRL.FilterIDs {
rp.FilterIDs = append(rp.FilterIDs, fltr)
for i, fltr := range tpRL.FilterIDs {
rp.FilterIDs[i] = fltr
}
for _, th := range tpRL.ThresholdIDs {
rp.ThresholdIDs = append(rp.ThresholdIDs, th)
for i, th := range tpRL.ThresholdIDs {
rp.ThresholdIDs[i] = th
}
if tpRL.ActivationInterval != nil {
if rp.ActivationInterval, err = tpRL.ActivationInterval.AsActivationInterval(timezone); err != nil {
@@ -2139,25 +2141,27 @@ func APItoModelStats(st *utils.TPStats) (mdls TpStatsS) {
func APItoStats(tpST *utils.TPStats, timezone string) (st *StatQueueProfile, err error) {
st = &StatQueueProfile{
Tenant: tpST.Tenant,
ID: tpST.ID,
QueueLength: tpST.QueueLength,
Metrics: tpST.Metrics,
Weight: tpST.Weight,
Blocker: tpST.Blocker,
Stored: tpST.Stored,
MinItems: tpST.MinItems,
Tenant: tpST.Tenant,
ID: tpST.ID,
QueueLength: tpST.QueueLength,
Metrics: tpST.Metrics,
Weight: tpST.Weight,
Blocker: tpST.Blocker,
Stored: tpST.Stored,
MinItems: tpST.MinItems,
ThresholdIDs: make([]string, len(tpST.ThresholdIDs)),
FilterIDs: make([]string, len(tpST.FilterIDs)),
}
if tpST.TTL != "" {
if st.TTL, err = utils.ParseDurationWithNanosecs(tpST.TTL); err != nil {
return nil, err
}
}
for _, trh := range tpST.ThresholdIDs {
st.ThresholdIDs = append(st.ThresholdIDs, trh)
for i, trh := range tpST.ThresholdIDs {
st.ThresholdIDs[i] = trh
}
for _, fltr := range tpST.FilterIDs {
st.FilterIDs = append(st.FilterIDs, fltr)
for i, fltr := range tpST.FilterIDs {
st.FilterIDs[i] = fltr
}
if tpST.ActivationInterval != nil {
if st.ActivationInterval, err = tpST.ActivationInterval.AsActivationInterval(timezone); err != nil {
@@ -2316,25 +2320,27 @@ func APItoModelTPThreshold(th *utils.TPThreshold) (mdls TpThresholdS) {
func APItoThresholdProfile(tpTH *utils.TPThreshold, timezone string) (th *ThresholdProfile, err error) {
th = &ThresholdProfile{
Tenant: tpTH.Tenant,
ID: tpTH.ID,
MaxHits: tpTH.MaxHits,
MinHits: tpTH.MinHits,
Weight: tpTH.Weight,
Blocker: tpTH.Blocker,
Async: tpTH.Async,
Tenant: tpTH.Tenant,
ID: tpTH.ID,
MaxHits: tpTH.MaxHits,
MinHits: tpTH.MinHits,
Weight: tpTH.Weight,
Blocker: tpTH.Blocker,
Async: tpTH.Async,
ActionIDs: make([]string, len(tpTH.ActionIDs)),
FilterIDs: make([]string, len(tpTH.FilterIDs)),
}
if tpTH.MinSleep != "" {
if th.MinSleep, err = utils.ParseDurationWithNanosecs(tpTH.MinSleep); err != nil {
return nil, err
}
}
for _, ati := range tpTH.ActionIDs {
th.ActionIDs = append(th.ActionIDs, ati)
for i, ati := range tpTH.ActionIDs {
th.ActionIDs[i] = ati
}
for _, fli := range tpTH.FilterIDs {
th.FilterIDs = append(th.FilterIDs, fli)
for i, fli := range tpTH.FilterIDs {
th.FilterIDs[i] = fli
}
if tpTH.ActivationInterval != nil {
if th.ActivationInterval, err = tpTH.ActivationInterval.AsActivationInterval(timezone); err != nil {
@@ -2639,27 +2645,29 @@ func APItoModelTPSuppliers(st *utils.TPSupplierProfile) (mdls TpSuppliers) {
return
}
func APItoSupplierProfile(tpTH *utils.TPSupplierProfile, timezone string) (th *SupplierProfile, err error) {
th = &SupplierProfile{
Tenant: tpTH.Tenant,
ID: tpTH.ID,
Sorting: tpTH.Sorting,
Weight: tpTH.Weight,
Suppliers: make([]*Supplier, len(tpTH.Suppliers)),
func APItoSupplierProfile(tpSPP *utils.TPSupplierProfile, timezone string) (spp *SupplierProfile, err error) {
spp = &SupplierProfile{
Tenant: tpSPP.Tenant,
ID: tpSPP.ID,
Sorting: tpSPP.Sorting,
Weight: tpSPP.Weight,
Suppliers: make([]*Supplier, len(tpSPP.Suppliers)),
SortingParameters: make([]string, len(tpSPP.SortingParameters)),
FilterIDs: make([]string, len(tpSPP.FilterIDs)),
}
for _, stp := range tpTH.SortingParameters {
th.SortingParameters = append(th.SortingParameters, stp)
for i, stp := range tpSPP.SortingParameters {
spp.SortingParameters[i] = stp
}
for _, fli := range tpTH.FilterIDs {
th.FilterIDs = append(th.FilterIDs, fli)
for i, fli := range tpSPP.FilterIDs {
spp.FilterIDs[i] = fli
}
if tpTH.ActivationInterval != nil {
if th.ActivationInterval, err = tpTH.ActivationInterval.AsActivationInterval(timezone); err != nil {
if tpSPP.ActivationInterval != nil {
if spp.ActivationInterval, err = tpSPP.ActivationInterval.AsActivationInterval(timezone); err != nil {
return nil, err
}
}
for i, suplier := range tpTH.Suppliers {
supl := &Supplier{
for i, suplier := range tpSPP.Suppliers {
spp.Suppliers[i] = &Supplier{
ID: suplier.ID,
Weight: suplier.Weight,
Blocker: suplier.Blocker,
@@ -2669,9 +2677,8 @@ func APItoSupplierProfile(tpTH *utils.TPSupplierProfile, timezone string) (th *S
StatIDs: suplier.StatIDs,
SupplierParameters: suplier.SupplierParameters,
}
th.Suppliers[i] = supl
}
return th, nil
return spp, nil
}
type TPAttributes []*TPAttribute
@@ -2791,39 +2798,40 @@ func APItoModelTPAttribute(th *utils.TPAttributeProfile) (mdls TPAttributes) {
return
}
func APItoAttributeProfile(tpTH *utils.TPAttributeProfile, timezone string) (th *AttributeProfile, err error) {
th = &AttributeProfile{
Tenant: tpTH.Tenant,
ID: tpTH.ID,
Weight: tpTH.Weight,
Blocker: tpTH.Blocker,
FilterIDs: []string{},
Contexts: []string{},
func APItoAttributeProfile(tpAttr *utils.TPAttributeProfile, timezone string) (attrPrf *AttributeProfile, err error) {
attrPrf = &AttributeProfile{
Tenant: tpAttr.Tenant,
ID: tpAttr.ID,
Weight: tpAttr.Weight,
Blocker: tpAttr.Blocker,
FilterIDs: make([]string, len(tpAttr.FilterIDs)),
Contexts: make([]string, len(tpAttr.Contexts)),
Attributes: make([]*Attribute, len(tpAttr.Attributes)),
}
for _, fli := range tpTH.FilterIDs {
th.FilterIDs = append(th.FilterIDs, fli)
for i, fli := range tpAttr.FilterIDs {
attrPrf.FilterIDs[i] = fli
}
for _, context := range tpTH.Contexts {
th.Contexts = append(th.Contexts, context)
for i, context := range tpAttr.Contexts {
attrPrf.Contexts[i] = context
}
for _, reqAttr := range tpTH.Attributes {
for i, reqAttr := range tpAttr.Attributes {
sbstPrsr, err := utils.NewRSRParsers(reqAttr.Substitute, true)
if err != nil {
return nil, err
}
th.Attributes = append(th.Attributes, &Attribute{
attrPrf.Attributes[i] = &Attribute{
Append: reqAttr.Append,
FieldName: reqAttr.FieldName,
Initial: reqAttr.Initial,
Substitute: sbstPrsr,
})
}
}
if tpTH.ActivationInterval != nil {
if th.ActivationInterval, err = tpTH.ActivationInterval.AsActivationInterval(timezone); err != nil {
if tpAttr.ActivationInterval != nil {
if attrPrf.ActivationInterval, err = tpAttr.ActivationInterval.AsActivationInterval(timezone); err != nil {
return nil, err
}
}
return th, nil
return attrPrf, nil
}
type TPChargers []*TPCharger
@@ -2980,14 +2988,14 @@ func APItoChargerProfile(tpCPP *utils.TPChargerProfile, timezone string) (cpp *C
ID: tpCPP.ID,
Weight: tpCPP.Weight,
RunID: tpCPP.RunID,
FilterIDs: []string{},
AttributeIDs: []string{},
FilterIDs: make([]string, len(tpCPP.FilterIDs)),
AttributeIDs: make([]string, len(tpCPP.AttributeIDs)),
}
for _, fli := range tpCPP.FilterIDs {
cpp.FilterIDs = append(cpp.FilterIDs, fli)
for i, fli := range tpCPP.FilterIDs {
cpp.FilterIDs[i] = fli
}
for _, attribute := range tpCPP.AttributeIDs {
cpp.AttributeIDs = append(cpp.AttributeIDs, attribute)
for i, attribute := range tpCPP.AttributeIDs {
cpp.AttributeIDs[i] = attribute
}
if tpCPP.ActivationInterval != nil {
if cpp.ActivationInterval, err = tpCPP.ActivationInterval.AsActivationInterval(timezone); err != nil {

16
loaders/loader_test.go
View File

@@ -311,6 +311,7 @@ cgrates.org,ResGroup2,*string:Account:1002,2014-07-29T15:00:00Z,3600s,2,premium_
Limit: 2,
Blocker: true,
Stored: true,
ThresholdIDs: []string{},
}
eResPrf2 := &engine.ResourceProfile{
Tenant: "cgrates.org",
@@ -325,6 +326,7 @@ cgrates.org,ResGroup2,*string:Account:1002,2014-07-29T15:00:00Z,3600s,2,premium_
Limit: 2,
Blocker: true,
Stored: true,
ThresholdIDs: []string{},
}
if len(ldr.bufLoaderData) != 0 {
t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData)
@@ -848,12 +850,14 @@ cgrates.org,SPL_LEASTCOST_1,,,,,supplier2,,,RP_RETAIL1,resource_spl2,,20,,,
t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData)
}
eSp1 := &engine.SupplierProfile{
Tenant: "cgrates.org",
ID: "SPL_WEIGHT_2",
Tenant: "cgrates.org",
ID: "SPL_WEIGHT_2",
FilterIDs: []string{},
ActivationInterval: &utils.ActivationInterval{
ActivationTime: time.Date(2017, 11, 27, 0, 0, 0, 0, time.UTC),
},
Sorting: "*weight",
Sorting: "*weight",
SortingParameters: []string{},
Suppliers: []*engine.Supplier{
&engine.Supplier{
ID: "supplier1",
@@ -869,7 +873,8 @@ cgrates.org,SPL_LEASTCOST_1,,,,,supplier2,,,RP_RETAIL1,resource_spl2,,20,,,
ActivationInterval: &utils.ActivationInterval{
ActivationTime: time.Date(2017, 11, 27, 0, 0, 0, 0, time.UTC),
},
Sorting: "*least_cost",
Sorting: "*least_cost",
SortingParameters: []string{},
Suppliers: []*engine.Supplier{
&engine.Supplier{
ID: "supplier1",
@@ -894,7 +899,8 @@ cgrates.org,SPL_LEASTCOST_1,,,,,supplier2,,,RP_RETAIL1,resource_spl2,,20,,,
ActivationInterval: &utils.ActivationInterval{
ActivationTime: time.Date(2017, 11, 27, 0, 0, 0, 0, time.UTC),
},
Sorting: "*least_cost",
Sorting: "*least_cost",
SortingParameters: []string{},
Suppliers: []*engine.Supplier{
&engine.Supplier{
ID: "supplier2",

3
migrator/action_plan.go
View File

@@ -64,6 +64,9 @@ func (m *Migrator) migrateCurrentActionPlans() (err error) {
if err := m.dmOut.DataManager().DataDB().SetActionPlan(idg, acts, true, utils.NonTransactional); err != nil {
return err
}
if err := m.dmIN.DataManager().DataDB().RemoveActionPlan(idg, utils.NonTransactional); err != nil {
return err
}
m.stats[utils.ActionPlans] += 1
}
}

4
migrator/action_plan_it_test.go
View File

@@ -246,5 +246,9 @@ func testActPlnITMigrateAndMove(t *testing.T) {
} else if !reflect.DeepEqual(actPln.ActionTimings[0].Timing, result.ActionTimings[0].Timing) {
t.Errorf("Expecting: %+v, received: %+v", actPln.ActionTimings[0].Timing, result.ActionTimings[0].Timing)
}
result, err = actPlnMigrator.dmIN.DataManager().DataDB().GetActionPlan((*v1actPln)[0].Id, true, utils.NonTransactional)
if err != utils.ErrNotFound {
t.Error(err)
}
}
}

8
migrator/attributes.go
View File

@@ -20,7 +20,6 @@ package migrator
import (
"fmt"
//"log"
"strings"
"github.com/cgrates/cgrates/config"
@@ -63,6 +62,11 @@ func (m *Migrator) migrateCurrentAttributeProfile() (err error) {
if err := m.dmOut.DataManager().SetAttributeProfile(attrPrf, true); err != nil {
return err
}
if err := m.dmIN.DataManager().RemoveAttributeProfile(tenant,
idg, attrPrf.Contexts, utils.NonTransactional, false); err != nil {
return err
}
m.stats[utils.Attributes] += 1
}
}
}
@@ -97,7 +101,7 @@ func (m *Migrator) migrateV1Attributes() (err error) {
}
if m.dryRun != true {
// All done, update version wtih current one
vrs := engine.Versions{utils.Attributes: engine.CurrentStorDBVersions()[utils.Attributes]}
vrs := engine.Versions{utils.Attributes: engine.CurrentDataDBVersions()[utils.Attributes]}
if err = m.dmOut.DataManager().DataDB().SetVersions(vrs, false); err != nil {
return utils.NewCGRError(utils.Migrator,
utils.ServerErrorCaps,

57
migrator/attributes_it_test.go
View File

@@ -81,7 +81,7 @@ func TestAttributeITMongo(t *testing.T) {
}
}
func TestAttributeITMove(t *testing.T) {
func TestAttributeITMove1(t *testing.T) {
var err error
attrPathIn = path.Join(*dataDir, "conf", "samples", "tutmongo")
attrCfgIn, err = config.NewCGRConfigFromFolder(attrPathIn)
@@ -99,6 +99,24 @@ func TestAttributeITMove(t *testing.T) {
}
}
func TestAttributeITMove2(t *testing.T) {
var err error
attrPathIn = path.Join(*dataDir, "conf", "samples", "tutmysql")
attrCfgIn, err = config.NewCGRConfigFromFolder(attrPathIn)
if err != nil {
t.Fatal(err)
}
attrPathOut = path.Join(*dataDir, "conf", "samples", "tutmongo")
attrCfgOut, err = config.NewCGRConfigFromFolder(attrPathOut)
if err != nil {
t.Fatal(err)
}
attrAction = utils.Move
for _, stest := range sTestsAttrIT {
t.Run("TestAttributeITMove", stest)
}
}
func TestAttributeITMoveEncoding(t *testing.T) {
var err error
attrPathIn = path.Join(*dataDir, "conf", "samples", "tutmongo")
@@ -159,9 +177,21 @@ func testAttrITConnect(t *testing.T) {
}
func testAttrITFlush(t *testing.T) {
attrMigrator.dmOut.DataManager().DataDB().Flush("")
if err := engine.SetDBVersions(attrMigrator.dmOut.DataManager().DataDB()); err != nil {
t.Error("Error ", err.Error())
if err := attrMigrator.dmOut.DataManager().DataDB().Flush(""); err != nil {
t.Error(err)
}
if isEmpty, err := attrMigrator.dmOut.DataManager().DataDB().IsDBEmpty(); err != nil {
t.Error(err)
} else if isEmpty != true {
t.Errorf("\nExpecting: true got :%+v", isEmpty)
}
if err := attrMigrator.dmIN.DataManager().DataDB().Flush(""); err != nil {
t.Error(err)
}
if isEmpty, err := attrMigrator.dmIN.DataManager().DataDB().IsDBEmpty(); err != nil {
t.Error(err)
} else if isEmpty != true {
t.Errorf("\nExpecting: true got :%+v", isEmpty)
}
}
@@ -199,13 +229,13 @@ func testAttrITMigrateAndMove(t *testing.T) {
&engine.Attribute{
FieldName: "FL1",
Initial: "In1",
Substitute: utils.NewRSRParsersMustCompile("1001", true),
Substitute: utils.NewRSRParsersMustCompile("Al1", true),
Append: true,
},
},
Weight: 20,
}
switch accAction {
switch attrAction {
case utils.Migrate:
err := attrMigrator.dmIN.setV1AttributeProfile(v1Attribute)
if err != nil {
@@ -235,14 +265,14 @@ func testAttrITMigrateAndMove(t *testing.T) {
t.Errorf("Unexpected version returned: %d", vrs[utils.Attributes])
}
result, err := attrMigrator.dmOut.DataManager().GetAttributeProfile("cgrates.org",
"ATTR_1", false, utils.NonTransactional)
"ATTR_1", true, utils.NonTransactional)
if err != nil {
t.Error("Error when getting Attribute ", err.Error())
}
result.Compile()
attrPrf.Compile()
if !reflect.DeepEqual(result, attrPrf) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf, result)
t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(attrPrf), utils.ToJSON(result))
}
case utils.Move:
if err := attrMigrator.dmIN.DataManager().SetAttributeProfile(attrPrf, false); err != nil {
@@ -253,12 +283,19 @@ func testAttrITMigrateAndMove(t *testing.T) {
if err != nil {
t.Error("Error when setting version for Attributes ", err.Error())
}
_, err = attrMigrator.dmOut.DataManager().GetAttributeProfile("cgrates.org",
"ATTR_1", true, utils.NonTransactional)
if err != utils.ErrNotFound {
t.Error(err)
}
err, _ = attrMigrator.Migrate([]string{utils.MetaAttributes})
if err != nil {
t.Error("Error when migrating Attributes ", err.Error())
}
result, err := attrMigrator.dmOut.DataManager().GetAttributeProfile("cgrates.org",
"ATTR_1", false, utils.NonTransactional)
"ATTR_1", true, utils.NonTransactional)
if err != nil {
t.Error(err)
}
@@ -268,7 +305,7 @@ func testAttrITMigrateAndMove(t *testing.T) {
t.Errorf("Expecting: %+v, received: %+v", attrPrf, result)
}
result, err = attrMigrator.dmIN.DataManager().GetAttributeProfile("cgrates.org",
"ATTR_1", false, utils.NonTransactional)
"ATTR_1", true, utils.NonTransactional)
if err != utils.ErrNotFound {
t.Error(err)
}