From 3edc941adfd511836fdc75abc2cf04fc1bb5eea7 Mon Sep 17 00:00:00 2001 From: TeoV Date: Wed, 25 Jul 2018 09:48:56 -0400 Subject: [PATCH] Update migrator --- engine/model_helpers.go | 144 +++++++++++++++++--------------- loaders/loader_test.go | 16 ++-- migrator/action_plan.go | 3 + migrator/action_plan_it_test.go | 4 + migrator/attributes.go | 8 +- migrator/attributes_it_test.go | 57 ++++++++++--- 6 files changed, 147 insertions(+), 85 deletions(-) diff --git a/engine/model_helpers.go b/engine/model_helpers.go index 07341e93e..2f8dfff82 100644 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -1944,17 +1944,19 @@ func APItoResource(tpRL *utils.TPResource, timezone string) (rp *ResourceProfile Blocker: tpRL.Blocker, Stored: tpRL.Stored, AllocationMessage: tpRL.AllocationMessage, + ThresholdIDs: make([]string, len(tpRL.ThresholdIDs)), + FilterIDs: make([]string, len(tpRL.FilterIDs)), } if tpRL.UsageTTL != "" { if rp.UsageTTL, err = utils.ParseDurationWithNanosecs(tpRL.UsageTTL); err != nil { return nil, err } } - for _, fltr := range tpRL.FilterIDs { - rp.FilterIDs = append(rp.FilterIDs, fltr) + for i, fltr := range tpRL.FilterIDs { + rp.FilterIDs[i] = fltr } - for _, th := range tpRL.ThresholdIDs { - rp.ThresholdIDs = append(rp.ThresholdIDs, th) + for i, th := range tpRL.ThresholdIDs { + rp.ThresholdIDs[i] = th } if tpRL.ActivationInterval != nil { if rp.ActivationInterval, err = tpRL.ActivationInterval.AsActivationInterval(timezone); err != nil { @@ -2139,25 +2141,27 @@ func APItoModelStats(st *utils.TPStats) (mdls TpStatsS) { func APItoStats(tpST *utils.TPStats, timezone string) (st *StatQueueProfile, err error) { st = &StatQueueProfile{ - Tenant: tpST.Tenant, - ID: tpST.ID, - QueueLength: tpST.QueueLength, - Metrics: tpST.Metrics, - Weight: tpST.Weight, - Blocker: tpST.Blocker, - Stored: tpST.Stored, - MinItems: tpST.MinItems, + Tenant: tpST.Tenant, + ID: tpST.ID, + QueueLength: tpST.QueueLength, + Metrics: tpST.Metrics, + Weight: tpST.Weight, + Blocker: tpST.Blocker, + Stored: tpST.Stored, + MinItems: tpST.MinItems, + ThresholdIDs: make([]string, len(tpST.ThresholdIDs)), + FilterIDs: make([]string, len(tpST.FilterIDs)), } if tpST.TTL != "" { if st.TTL, err = utils.ParseDurationWithNanosecs(tpST.TTL); err != nil { return nil, err } } - for _, trh := range tpST.ThresholdIDs { - st.ThresholdIDs = append(st.ThresholdIDs, trh) + for i, trh := range tpST.ThresholdIDs { + st.ThresholdIDs[i] = trh } - for _, fltr := range tpST.FilterIDs { - st.FilterIDs = append(st.FilterIDs, fltr) + for i, fltr := range tpST.FilterIDs { + st.FilterIDs[i] = fltr } if tpST.ActivationInterval != nil { if st.ActivationInterval, err = tpST.ActivationInterval.AsActivationInterval(timezone); err != nil { @@ -2316,25 +2320,27 @@ func APItoModelTPThreshold(th *utils.TPThreshold) (mdls TpThresholdS) { func APItoThresholdProfile(tpTH *utils.TPThreshold, timezone string) (th *ThresholdProfile, err error) { th = &ThresholdProfile{ - Tenant: tpTH.Tenant, - ID: tpTH.ID, - MaxHits: tpTH.MaxHits, - MinHits: tpTH.MinHits, - Weight: tpTH.Weight, - Blocker: tpTH.Blocker, - Async: tpTH.Async, + Tenant: tpTH.Tenant, + ID: tpTH.ID, + MaxHits: tpTH.MaxHits, + MinHits: tpTH.MinHits, + Weight: tpTH.Weight, + Blocker: tpTH.Blocker, + Async: tpTH.Async, + ActionIDs: make([]string, len(tpTH.ActionIDs)), + FilterIDs: make([]string, len(tpTH.FilterIDs)), } if tpTH.MinSleep != "" { if th.MinSleep, err = utils.ParseDurationWithNanosecs(tpTH.MinSleep); err != nil { return nil, err } } - for _, ati := range tpTH.ActionIDs { - th.ActionIDs = append(th.ActionIDs, ati) + for i, ati := range tpTH.ActionIDs { + th.ActionIDs[i] = ati } - for _, fli := range tpTH.FilterIDs { - th.FilterIDs = append(th.FilterIDs, fli) + for i, fli := range tpTH.FilterIDs { + th.FilterIDs[i] = fli } if tpTH.ActivationInterval != nil { if th.ActivationInterval, err = tpTH.ActivationInterval.AsActivationInterval(timezone); err != nil { @@ -2639,27 +2645,29 @@ func APItoModelTPSuppliers(st *utils.TPSupplierProfile) (mdls TpSuppliers) { return } -func APItoSupplierProfile(tpTH *utils.TPSupplierProfile, timezone string) (th *SupplierProfile, err error) { - th = &SupplierProfile{ - Tenant: tpTH.Tenant, - ID: tpTH.ID, - Sorting: tpTH.Sorting, - Weight: tpTH.Weight, - Suppliers: make([]*Supplier, len(tpTH.Suppliers)), +func APItoSupplierProfile(tpSPP *utils.TPSupplierProfile, timezone string) (spp *SupplierProfile, err error) { + spp = &SupplierProfile{ + Tenant: tpSPP.Tenant, + ID: tpSPP.ID, + Sorting: tpSPP.Sorting, + Weight: tpSPP.Weight, + Suppliers: make([]*Supplier, len(tpSPP.Suppliers)), + SortingParameters: make([]string, len(tpSPP.SortingParameters)), + FilterIDs: make([]string, len(tpSPP.FilterIDs)), } - for _, stp := range tpTH.SortingParameters { - th.SortingParameters = append(th.SortingParameters, stp) + for i, stp := range tpSPP.SortingParameters { + spp.SortingParameters[i] = stp } - for _, fli := range tpTH.FilterIDs { - th.FilterIDs = append(th.FilterIDs, fli) + for i, fli := range tpSPP.FilterIDs { + spp.FilterIDs[i] = fli } - if tpTH.ActivationInterval != nil { - if th.ActivationInterval, err = tpTH.ActivationInterval.AsActivationInterval(timezone); err != nil { + if tpSPP.ActivationInterval != nil { + if spp.ActivationInterval, err = tpSPP.ActivationInterval.AsActivationInterval(timezone); err != nil { return nil, err } } - for i, suplier := range tpTH.Suppliers { - supl := &Supplier{ + for i, suplier := range tpSPP.Suppliers { + spp.Suppliers[i] = &Supplier{ ID: suplier.ID, Weight: suplier.Weight, Blocker: suplier.Blocker, @@ -2669,9 +2677,8 @@ func APItoSupplierProfile(tpTH *utils.TPSupplierProfile, timezone string) (th *S StatIDs: suplier.StatIDs, SupplierParameters: suplier.SupplierParameters, } - th.Suppliers[i] = supl } - return th, nil + return spp, nil } type TPAttributes []*TPAttribute @@ -2791,39 +2798,40 @@ func APItoModelTPAttribute(th *utils.TPAttributeProfile) (mdls TPAttributes) { return } -func APItoAttributeProfile(tpTH *utils.TPAttributeProfile, timezone string) (th *AttributeProfile, err error) { - th = &AttributeProfile{ - Tenant: tpTH.Tenant, - ID: tpTH.ID, - Weight: tpTH.Weight, - Blocker: tpTH.Blocker, - FilterIDs: []string{}, - Contexts: []string{}, +func APItoAttributeProfile(tpAttr *utils.TPAttributeProfile, timezone string) (attrPrf *AttributeProfile, err error) { + attrPrf = &AttributeProfile{ + Tenant: tpAttr.Tenant, + ID: tpAttr.ID, + Weight: tpAttr.Weight, + Blocker: tpAttr.Blocker, + FilterIDs: make([]string, len(tpAttr.FilterIDs)), + Contexts: make([]string, len(tpAttr.Contexts)), + Attributes: make([]*Attribute, len(tpAttr.Attributes)), } - for _, fli := range tpTH.FilterIDs { - th.FilterIDs = append(th.FilterIDs, fli) + for i, fli := range tpAttr.FilterIDs { + attrPrf.FilterIDs[i] = fli } - for _, context := range tpTH.Contexts { - th.Contexts = append(th.Contexts, context) + for i, context := range tpAttr.Contexts { + attrPrf.Contexts[i] = context } - for _, reqAttr := range tpTH.Attributes { + for i, reqAttr := range tpAttr.Attributes { sbstPrsr, err := utils.NewRSRParsers(reqAttr.Substitute, true) if err != nil { return nil, err } - th.Attributes = append(th.Attributes, &Attribute{ + attrPrf.Attributes[i] = &Attribute{ Append: reqAttr.Append, FieldName: reqAttr.FieldName, Initial: reqAttr.Initial, Substitute: sbstPrsr, - }) + } } - if tpTH.ActivationInterval != nil { - if th.ActivationInterval, err = tpTH.ActivationInterval.AsActivationInterval(timezone); err != nil { + if tpAttr.ActivationInterval != nil { + if attrPrf.ActivationInterval, err = tpAttr.ActivationInterval.AsActivationInterval(timezone); err != nil { return nil, err } } - return th, nil + return attrPrf, nil } type TPChargers []*TPCharger @@ -2980,14 +2988,14 @@ func APItoChargerProfile(tpCPP *utils.TPChargerProfile, timezone string) (cpp *C ID: tpCPP.ID, Weight: tpCPP.Weight, RunID: tpCPP.RunID, - FilterIDs: []string{}, - AttributeIDs: []string{}, + FilterIDs: make([]string, len(tpCPP.FilterIDs)), + AttributeIDs: make([]string, len(tpCPP.AttributeIDs)), } - for _, fli := range tpCPP.FilterIDs { - cpp.FilterIDs = append(cpp.FilterIDs, fli) + for i, fli := range tpCPP.FilterIDs { + cpp.FilterIDs[i] = fli } - for _, attribute := range tpCPP.AttributeIDs { - cpp.AttributeIDs = append(cpp.AttributeIDs, attribute) + for i, attribute := range tpCPP.AttributeIDs { + cpp.AttributeIDs[i] = attribute } if tpCPP.ActivationInterval != nil { if cpp.ActivationInterval, err = tpCPP.ActivationInterval.AsActivationInterval(timezone); err != nil { diff --git a/loaders/loader_test.go b/loaders/loader_test.go index c31ec66c1..db16daa11 100644 --- a/loaders/loader_test.go +++ b/loaders/loader_test.go @@ -311,6 +311,7 @@ cgrates.org,ResGroup2,*string:Account:1002,2014-07-29T15:00:00Z,3600s,2,premium_ Limit: 2, Blocker: true, Stored: true, + ThresholdIDs: []string{}, } eResPrf2 := &engine.ResourceProfile{ Tenant: "cgrates.org", @@ -325,6 +326,7 @@ cgrates.org,ResGroup2,*string:Account:1002,2014-07-29T15:00:00Z,3600s,2,premium_ Limit: 2, Blocker: true, Stored: true, + ThresholdIDs: []string{}, } if len(ldr.bufLoaderData) != 0 { t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData) @@ -848,12 +850,14 @@ cgrates.org,SPL_LEASTCOST_1,,,,,supplier2,,,RP_RETAIL1,resource_spl2,,20,,, t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData) } eSp1 := &engine.SupplierProfile{ - Tenant: "cgrates.org", - ID: "SPL_WEIGHT_2", + Tenant: "cgrates.org", + ID: "SPL_WEIGHT_2", + FilterIDs: []string{}, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2017, 11, 27, 0, 0, 0, 0, time.UTC), }, - Sorting: "*weight", + Sorting: "*weight", + SortingParameters: []string{}, Suppliers: []*engine.Supplier{ &engine.Supplier{ ID: "supplier1", @@ -869,7 +873,8 @@ cgrates.org,SPL_LEASTCOST_1,,,,,supplier2,,,RP_RETAIL1,resource_spl2,,20,,, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2017, 11, 27, 0, 0, 0, 0, time.UTC), }, - Sorting: "*least_cost", + Sorting: "*least_cost", + SortingParameters: []string{}, Suppliers: []*engine.Supplier{ &engine.Supplier{ ID: "supplier1", @@ -894,7 +899,8 @@ cgrates.org,SPL_LEASTCOST_1,,,,,supplier2,,,RP_RETAIL1,resource_spl2,,20,,, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2017, 11, 27, 0, 0, 0, 0, time.UTC), }, - Sorting: "*least_cost", + Sorting: "*least_cost", + SortingParameters: []string{}, Suppliers: []*engine.Supplier{ &engine.Supplier{ ID: "supplier2", diff --git a/migrator/action_plan.go b/migrator/action_plan.go index b4d770892..b2b822847 100644 --- a/migrator/action_plan.go +++ b/migrator/action_plan.go @@ -64,6 +64,9 @@ func (m *Migrator) migrateCurrentActionPlans() (err error) { if err := m.dmOut.DataManager().DataDB().SetActionPlan(idg, acts, true, utils.NonTransactional); err != nil { return err } + if err := m.dmIN.DataManager().DataDB().RemoveActionPlan(idg, utils.NonTransactional); err != nil { + return err + } m.stats[utils.ActionPlans] += 1 } } diff --git a/migrator/action_plan_it_test.go b/migrator/action_plan_it_test.go index 12e57186c..f9b014751 100644 --- a/migrator/action_plan_it_test.go +++ b/migrator/action_plan_it_test.go @@ -246,5 +246,9 @@ func testActPlnITMigrateAndMove(t *testing.T) { } else if !reflect.DeepEqual(actPln.ActionTimings[0].Timing, result.ActionTimings[0].Timing) { t.Errorf("Expecting: %+v, received: %+v", actPln.ActionTimings[0].Timing, result.ActionTimings[0].Timing) } + result, err = actPlnMigrator.dmIN.DataManager().DataDB().GetActionPlan((*v1actPln)[0].Id, true, utils.NonTransactional) + if err != utils.ErrNotFound { + t.Error(err) + } } } diff --git a/migrator/attributes.go b/migrator/attributes.go index 53b8a7722..450fe2827 100644 --- a/migrator/attributes.go +++ b/migrator/attributes.go @@ -20,7 +20,6 @@ package migrator import ( "fmt" - //"log" "strings" "github.com/cgrates/cgrates/config" @@ -63,6 +62,11 @@ func (m *Migrator) migrateCurrentAttributeProfile() (err error) { if err := m.dmOut.DataManager().SetAttributeProfile(attrPrf, true); err != nil { return err } + if err := m.dmIN.DataManager().RemoveAttributeProfile(tenant, + idg, attrPrf.Contexts, utils.NonTransactional, false); err != nil { + return err + } + m.stats[utils.Attributes] += 1 } } } @@ -97,7 +101,7 @@ func (m *Migrator) migrateV1Attributes() (err error) { } if m.dryRun != true { // All done, update version wtih current one - vrs := engine.Versions{utils.Attributes: engine.CurrentStorDBVersions()[utils.Attributes]} + vrs := engine.Versions{utils.Attributes: engine.CurrentDataDBVersions()[utils.Attributes]} if err = m.dmOut.DataManager().DataDB().SetVersions(vrs, false); err != nil { return utils.NewCGRError(utils.Migrator, utils.ServerErrorCaps, diff --git a/migrator/attributes_it_test.go b/migrator/attributes_it_test.go index a4e284604..00522a622 100755 --- a/migrator/attributes_it_test.go +++ b/migrator/attributes_it_test.go @@ -81,7 +81,7 @@ func TestAttributeITMongo(t *testing.T) { } } -func TestAttributeITMove(t *testing.T) { +func TestAttributeITMove1(t *testing.T) { var err error attrPathIn = path.Join(*dataDir, "conf", "samples", "tutmongo") attrCfgIn, err = config.NewCGRConfigFromFolder(attrPathIn) @@ -99,6 +99,24 @@ func TestAttributeITMove(t *testing.T) { } } +func TestAttributeITMove2(t *testing.T) { + var err error + attrPathIn = path.Join(*dataDir, "conf", "samples", "tutmysql") + attrCfgIn, err = config.NewCGRConfigFromFolder(attrPathIn) + if err != nil { + t.Fatal(err) + } + attrPathOut = path.Join(*dataDir, "conf", "samples", "tutmongo") + attrCfgOut, err = config.NewCGRConfigFromFolder(attrPathOut) + if err != nil { + t.Fatal(err) + } + attrAction = utils.Move + for _, stest := range sTestsAttrIT { + t.Run("TestAttributeITMove", stest) + } +} + func TestAttributeITMoveEncoding(t *testing.T) { var err error attrPathIn = path.Join(*dataDir, "conf", "samples", "tutmongo") @@ -159,9 +177,21 @@ func testAttrITConnect(t *testing.T) { } func testAttrITFlush(t *testing.T) { - attrMigrator.dmOut.DataManager().DataDB().Flush("") - if err := engine.SetDBVersions(attrMigrator.dmOut.DataManager().DataDB()); err != nil { - t.Error("Error ", err.Error()) + if err := attrMigrator.dmOut.DataManager().DataDB().Flush(""); err != nil { + t.Error(err) + } + if isEmpty, err := attrMigrator.dmOut.DataManager().DataDB().IsDBEmpty(); err != nil { + t.Error(err) + } else if isEmpty != true { + t.Errorf("\nExpecting: true got :%+v", isEmpty) + } + if err := attrMigrator.dmIN.DataManager().DataDB().Flush(""); err != nil { + t.Error(err) + } + if isEmpty, err := attrMigrator.dmIN.DataManager().DataDB().IsDBEmpty(); err != nil { + t.Error(err) + } else if isEmpty != true { + t.Errorf("\nExpecting: true got :%+v", isEmpty) } } @@ -199,13 +229,13 @@ func testAttrITMigrateAndMove(t *testing.T) { &engine.Attribute{ FieldName: "FL1", Initial: "In1", - Substitute: utils.NewRSRParsersMustCompile("1001", true), + Substitute: utils.NewRSRParsersMustCompile("Al1", true), Append: true, }, }, Weight: 20, } - switch accAction { + switch attrAction { case utils.Migrate: err := attrMigrator.dmIN.setV1AttributeProfile(v1Attribute) if err != nil { @@ -235,14 +265,14 @@ func testAttrITMigrateAndMove(t *testing.T) { t.Errorf("Unexpected version returned: %d", vrs[utils.Attributes]) } result, err := attrMigrator.dmOut.DataManager().GetAttributeProfile("cgrates.org", - "ATTR_1", false, utils.NonTransactional) + "ATTR_1", true, utils.NonTransactional) if err != nil { t.Error("Error when getting Attribute ", err.Error()) } result.Compile() attrPrf.Compile() if !reflect.DeepEqual(result, attrPrf) { - t.Errorf("Expecting: %+v, received: %+v", attrPrf, result) + t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(attrPrf), utils.ToJSON(result)) } case utils.Move: if err := attrMigrator.dmIN.DataManager().SetAttributeProfile(attrPrf, false); err != nil { @@ -253,12 +283,19 @@ func testAttrITMigrateAndMove(t *testing.T) { if err != nil { t.Error("Error when setting version for Attributes ", err.Error()) } + + _, err = attrMigrator.dmOut.DataManager().GetAttributeProfile("cgrates.org", + "ATTR_1", true, utils.NonTransactional) + if err != utils.ErrNotFound { + t.Error(err) + } + err, _ = attrMigrator.Migrate([]string{utils.MetaAttributes}) if err != nil { t.Error("Error when migrating Attributes ", err.Error()) } result, err := attrMigrator.dmOut.DataManager().GetAttributeProfile("cgrates.org", - "ATTR_1", false, utils.NonTransactional) + "ATTR_1", true, utils.NonTransactional) if err != nil { t.Error(err) } @@ -268,7 +305,7 @@ func testAttrITMigrateAndMove(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", attrPrf, result) } result, err = attrMigrator.dmIN.DataManager().GetAttributeProfile("cgrates.org", - "ATTR_1", false, utils.NonTransactional) + "ATTR_1", true, utils.NonTransactional) if err != utils.ErrNotFound { t.Error(err) }