From 180be2f24ff720b74f0cf404be9593ce85aeb8ed Mon Sep 17 00:00:00 2001 From: adragusin Date: Wed, 25 Mar 2020 15:53:26 +0200 Subject: [PATCH] Updated migrator and integration tests for attributes --- migrator/attributes.go | 101 +++++++--------------- migrator/attributes_it_test.go | 147 ++++++++++++++++----------------- 2 files changed, 98 insertions(+), 150 deletions(-) diff --git a/migrator/attributes.go b/migrator/attributes.go index 8325a46e1..87483e7d8 100644 --- a/migrator/attributes.go +++ b/migrator/attributes.go @@ -117,35 +117,26 @@ func (m *Migrator) migrateV1Attributes() (err error) { return } -func (m *Migrator) migrateV1ToV2Attributes() (v2Attr []*v2AttributeProfile, err error) { +func (m *Migrator) migrateV1ToV2Attributes() (v2Attr *v2AttributeProfile, err error) { var v1Attr *v1AttributeProfile - var attr *v2AttributeProfile - for { - v1Attr, err = m.dmIN.getV1AttributeProfile() - if err != nil && err != utils.ErrNoMoreData { - return nil, err - } - if err == utils.ErrNoMoreData { - break - } - if v1Attr == nil { - continue - } - attr, err = v1Attr.AsAttributeProfileV2() - if err != nil { - return nil, err - } - v2Attr = append(v2Attr, attr) - if m.dryRun { - continue - } + + v1Attr, err = m.dmIN.getV1AttributeProfile() + if err != nil { + return nil, err + } else if v1Attr == nil { + return nil, errors.New("Attribute NIL") } + + v2Attr, err = v1Attr.AsAttributeProfileV2() + if err != nil { + return nil, err + } + return } func (m *Migrator) migrateV1ToV4AttributeProfile() (v4Attr *v4AttributeProfile, err error) { var v1Attr *v1AttributeProfile - v1Attr, err = m.dmIN.getV1AttributeProfile() if err != nil { return nil, err @@ -328,49 +319,6 @@ func (m *Migrator) migrateV4ToV5AttributeProfile(v4Attr *v4AttributeProfile) (v5 func (m *Migrator) migrateAttributeProfile() (err error) { var vrs engine.Versions current := engine.CurrentDataDBVersions() - vrs, err = m.dmIN.DataManager().DataDB().GetVersions("") - if err != nil { - return utils.NewCGRError(utils.Migrator, - utils.ServerErrorCaps, - err.Error(), - fmt.Sprintf("error: <%s> when querying oldDataDB for versions", err.Error())) - } else if len(vrs) == 0 { - return utils.NewCGRError(utils.Migrator, - utils.MandatoryIEMissingCaps, - utils.UndefinedVersion, - "version number is not defined for ActionTriggers model") - } - switch vrs[utils.Attributes] { - case current[utils.Attributes]: - if m.sameDataDB { - break - } - if err = m.migrateCurrentAttributeProfile(); err != nil { - return err - } - case 1: - if err = m.migrateV1Attributes(); err != nil { - return err - } - case 2: - if err = m.migrateV2Attributes(); err != nil { - return err - } - case 3: - if err = m.migrateV3Attributes(); err != nil { - return err - } - case 4: - if err = m.migrateV4Attributes(); err != nil { - return err - } - } - return m.ensureIndexesDataDB(engine.ColAttr) -} - -//migrateAttrProfile step-by-step -func (m *Migrator) migrateAttributeProfileV2() (err error) { - var vrs engine.Versions vrs, err = m.dmIN.DataManager().DataDB().GetVersions(utils.EmptyString) if err != nil { //error getting the current verions @@ -381,17 +329,27 @@ func (m *Migrator) migrateAttributeProfileV2() (err error) { utils.UndefinedVersion, "version number is not defined for ActionTriggers model") } + migrated := true var v2Attr *v2AttributeProfile var v3Attr *v3AttributeProfile var v4Attr *v4AttributeProfile var v5Attr *engine.AttributeProfile - for { // One attribute profile at a time version := vrs[utils.Attributes] for { //Keep migrating until Attribute Profile reaches latest version switch version { + case current[utils.Attributes]: + if m.sameDataDB { + break + } + if err = m.migrateCurrentAttributeProfile(); err != nil { + return err + } + version = 5 + migrated = false + break case 1: // Migrate from V1 to V4 if v4Attr, err = m.migrateV1ToV4AttributeProfile(); err != nil && err != utils.ErrNoMoreData { return err @@ -420,26 +378,25 @@ func (m *Migrator) migrateAttributeProfileV2() (err error) { if v5Attr, err = m.migrateV4ToV5AttributeProfile(v4Attr); err != nil && err != utils.ErrNoMoreData { return err } else if err == utils.ErrNoMoreData { - break } version = 5 } - if version == 5 || err == utils.ErrNoMoreData { + if version == current[utils.Attributes] || err == utils.ErrNoMoreData { break } } - if err == utils.ErrNoMoreData { + if err == utils.ErrNoMoreData || !migrated { break } - if !m.dryRun { + if !m.dryRun && migrated { if vrs[utils.Attributes] == 1 { if err := m.dmOut.DataManager().DataDB().SetAttributeProfileDrv(v5Attr); err != nil { return err } } - //Set the fresh-migrated AttributeProfile into DB + // Set the fresh-migrated AttributeProfile into DB if err := m.dmOut.DataManager().SetAttributeProfile(v5Attr, true); err != nil { return err } @@ -447,7 +404,7 @@ func (m *Migrator) migrateAttributeProfileV2() (err error) { m.stats[utils.Attributes]++ } - if m.dryRun { + if m.dryRun || !migrated { return nil } // All done, update version wtih current one diff --git a/migrator/attributes_it_test.go b/migrator/attributes_it_test.go index a93e0110b..eb0f556f1 100755 --- a/migrator/attributes_it_test.go +++ b/migrator/attributes_it_test.go @@ -53,12 +53,12 @@ var sTestsAttrIT = []func(t *testing.T){ testAttrITMigrateV3, testAttrITFlush, testAttrITMigrateV4, - // testAttrITFlush, - // testAttrITV1ToV2, testAttrITFlush, testAttrITV1ToV5, testAttrITFlush, testAttrITV2ToV5, + testAttrITFlush, + testAttrITdryRunV2ToV5, } func TestAttributeITRedis(t *testing.T) { @@ -672,68 +672,8 @@ func testAttrITMigrateV4(t *testing.T) { } } -func testAttrITV1ToV2(t *testing.T) { - mapSubstitutes := make(map[string]map[string]*v1Attribute) - mapSubstitutes["FL1"] = make(map[string]*v1Attribute) - mapSubstitutes["FL1"]["In1"] = &v1Attribute{ - FieldName: "FL1", - Initial: "In1", - Substitute: "Al1", - Append: true, - } - v1Attribute := &v1AttributeProfile{ - Tenant: "cgrates.org", - ID: "attributeprofile1", - Contexts: []string{utils.MetaSessionS}, - FilterIDs: []string{"filter1"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), - ExpiryTime: time.Date(2020, 4, 18, 14, 25, 0, 0, time.UTC), - }, - Attributes: mapSubstitutes, - Weight: 20, - } - - //set attribute into inDB - - attrMigrator.dmIN.setV1AttributeProfile(v1Attribute) - - sbstPrsr, err := config.NewRSRParsers("Al1", true, config.CgrConfig().GeneralCfg().RSRSep) - if err != nil { - t.Error("Error converting Substitute from string to RSRParser: ", err) - } - eOut := []*v2AttributeProfile{ - &v2AttributeProfile{ - Tenant: "cgrates.org", - ID: "attributeprofile1", - Contexts: []string{utils.MetaSessionS}, - FilterIDs: []string{"filter1"}, - ActivationInterval: &utils.ActivationInterval{ - ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), - ExpiryTime: time.Date(2020, 4, 18, 14, 25, 0, 0, time.UTC), - }, - Attributes: []*v2Attribute{ - &v2Attribute{ - FieldName: "FL1", - Initial: "In1", - Substitute: sbstPrsr, - Append: true, - }, - }, - Weight: 20, - }, - } - - if v2, err := attrMigrator.migrateV1ToV2Attributes(); err != nil { - t.Error(err) - } else if !reflect.DeepEqual(eOut, v2) { - t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(eOut), utils.ToJSON(v2)) - } - -} - func testAttrITV1ToV5(t *testing.T) { - // contruct the v1 attribute with all fields filled up + // contruct the first v1 attributeProfile with all fields filled up mapSubstitutes := make(map[string]map[string]*v1Attribute) mapSubstitutes["FL1"] = make(map[string]*v1Attribute) mapSubstitutes["FL1"]["In1"] = &v1Attribute{ @@ -754,7 +694,7 @@ func testAttrITV1ToV5(t *testing.T) { Attributes: mapSubstitutes, Weight: 20, } - //second attribute profile + // contruct the second v1 attributeProfile with all fields filled up v1AttributeProfile2 := &v1AttributeProfile{ Tenant: "cgrates.org", ID: "attributeprofile2", @@ -768,16 +708,17 @@ func testAttrITV1ToV5(t *testing.T) { Weight: 20, } - //set attribute into inDB + // set the first attributeProfile into DB attrMigrator.dmIN.setV1AttributeProfile(v1AttributeProfile1) - //set the second attr profile + // set the second attributeProfile into DB attrMigrator.dmIN.setV1AttributeProfile(v1AttributeProfile2) - //set attributes version into DB + // set attributes version into DB if err := attrMigrator.dmIN.DataManager().DataDB().SetVersions(engine.Versions{utils.Attributes: 1}, true); err != nil { t.Errorf("error: <%s> when updating Attributes version into dataDB", err.Error()) } + // Construct the exepected output sbstPrsr, err := config.NewRSRParsers("Al1", true, config.CgrConfig().GeneralCfg().RSRSep) if err != nil { t.Error("Error converting Substitute from string to RSRParser: ", err) @@ -819,22 +760,22 @@ func testAttrITV1ToV5(t *testing.T) { Weight: 20, } - //Migrate to latest version - if err := attrMigrator.migrateAttributeProfileV2(); err != nil { + // Migrate to latest version + if err := attrMigrator.migrateAttributeProfile(); err != nil { t.Error(err) } - //check the version + // check the version if vrs, err := attrMigrator.dmOut.DataManager().DataDB().GetVersions(""); err != nil { t.Error(err) } else if vrs[utils.Attributes] != 5 { t.Errorf("Unexpected version returned: %d", vrs[utils.Attributes]) } - //check the first AttributeProfile + // check the first AttributeProfile result, err := attrMigrator.dmOut.DataManager().GetAttributeProfile("cgrates.org", "attributeprofile1", false, false, utils.NonTransactional) if err != nil { - t.Error(err) //only encoded map or array can be decoded into a struct + t.Error(err) } else { result.Compile() @@ -843,7 +784,7 @@ func testAttrITV1ToV5(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(eOut1), utils.ToJSON(result)) } } - //check the second AttributeProfile + // check the second AttributeProfile result, err = attrMigrator.dmOut.DataManager().GetAttributeProfile("cgrates.org", "attributeprofile2", false, false, utils.NonTransactional) if err != nil { @@ -858,7 +799,7 @@ func testAttrITV1ToV5(t *testing.T) { } func testAttrITV2ToV5(t *testing.T) { - // contruct the v1 attribute profile with all fields filled up + // contruct the first v2 attributeProfile with all fields filled up sbstPrsr, err := config.NewRSRParsers("Al1", true, config.CgrConfig().GeneralCfg().RSRSep) if err != nil { t.Error("Error converting Substitute from string to RSRParser: ", err) @@ -881,7 +822,7 @@ func testAttrITV2ToV5(t *testing.T) { }}, Weight: 20, } - //second attribute profile + // contruct the second v2 attributeProfile with all fields filled up v2AttributeProfile2 := &v2AttributeProfile{ Tenant: "cgrates.org", ID: "attributeprofile2", @@ -901,9 +842,9 @@ func testAttrITV2ToV5(t *testing.T) { Weight: 20, } - //set attribute into inDB + // set the first attributeProfile into inDB attrMigrator.dmIN.setV2AttributeProfile(v2AttributeProfile1) - //set the second attr profile + // set the second attributeProfile into inDB attrMigrator.dmIN.setV2AttributeProfile(v2AttributeProfile2) //set attributes version into DB @@ -911,6 +852,7 @@ func testAttrITV2ToV5(t *testing.T) { t.Errorf("error: <%s> when updating Attributes version into dataDB", err.Error()) } + // Construct the expected output eOut1 := &engine.AttributeProfile{ Tenant: "cgrates.org", ID: "attributeprofile1", @@ -949,7 +891,7 @@ func testAttrITV2ToV5(t *testing.T) { } //Migrate to latest version - if err := attrMigrator.migrateAttributeProfileV2(); err != nil { + if err := attrMigrator.migrateAttributeProfile(); err != nil { t.Error(err) } //check the version @@ -985,3 +927,52 @@ func testAttrITV2ToV5(t *testing.T) { } } } + +func testAttrITdryRunV2ToV5(t *testing.T) { + // Test with dryRun on true + // contruct the v2 attributeProfile with all fields filled up + sbstPrsr, err := config.NewRSRParsers("Al1", true, config.CgrConfig().GeneralCfg().RSRSep) + if err != nil { + t.Error("Error converting Substitute from string to RSRParser: ", err) + } + v2AttributeProfile := &v2AttributeProfile{ + Tenant: "cgrates.org", + ID: "attributeprofile1", + Contexts: []string{utils.MetaSessionS}, + FilterIDs: []string{"*string:test:test"}, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC), + ExpiryTime: time.Date(2020, 4, 18, 14, 25, 0, 0, time.UTC), + }, + Attributes: []*v2Attribute{ + &v2Attribute{ + FieldName: "FL1", + Initial: "In1", + Substitute: sbstPrsr, + Append: true, + }}, + Weight: 20, + } + //set dryRun on true + attrMigrator.dryRun = true + //set attributeProfile into inDB + attrMigrator.dmIN.setV2AttributeProfile(v2AttributeProfile) + + //set attributes version into DB + if err := attrMigrator.dmIN.DataManager().DataDB().SetVersions(engine.Versions{utils.Attributes: 2}, true); err != nil { + t.Errorf("error: <%s> when updating Attributes version into dataDB", err.Error()) + } + + // Should migrate with no errors and no data modified + if err := attrMigrator.migrateAttributeProfile(); err != nil { + t.Error(err) + } + + // Check if the attribute profile was set into DB + // Expecting NOT_FOUND as it was a dryRun migration + _, err = attrMigrator.dmOut.DataManager().GetAttributeProfile("cgrates.org", + "attributeprofile1", false, false, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + t.Errorf("Expecting: %+v, received: %+v", utils.ErrNotFound, err) + } +}