Updated filters migration

apier/v1/stats_it_test.go

@@ -374,10 +374,10 @@ func testV1STSSetStatQueueProfile(t *testing.T) {
 			QueueLength: 10,
 			TTL:         time.Duration(10) * time.Second,
 			Metrics: []*engine.MetricWithFilters{
-				&engine.MetricWithFilters{
+				{
 					MetricID: utils.MetaACD,
 				},
-				&engine.MetricWithFilters{
+				{
 					MetricID: utils.MetaTCD,
 				},
 			},

config/rsrparser.go

@@ -222,6 +222,7 @@ func (prsr *RSRParser) Compile() (err error) {
 	// dynamic content via attributeNames
 	spltRules := spltRgxp.Split(parserRules, -1)
 	prsr.path = spltRules[0] // in form ~hdr_name
+	prsr.rsrRules = make([]*utils.ReSearchReplace, 0, len(spltRules[1:]))
 	if len(spltRules) > 1 {
 		for _, ruleStr := range spltRules[1:] { // :s/ already removed through split
 			allMatches := rulesRgxp.FindStringSubmatch(ruleStr)
@@ -256,11 +257,7 @@ func (prsr *RSRParser) parseValue(value string) (out string, err error) {
 	for _, rsRule := range prsr.rsrRules {
 		value = rsRule.Process(value)
 	}
-
-	if out, err = prsr.converters.ConvertString(value); err != nil {
-		return
-	}
-	return
+	return prsr.converters.ConvertString(value)
 }
 
 // ParseValue will parse the value out considering converters

config/rsrparser_test.go

@@ -31,7 +31,7 @@ func TestNewRSRParsers(t *testing.T) {
 	eRSRParsers := RSRParsers{
 		&RSRParser{Rules: "Value1", path: "Value1"},
 		&RSRParser{Rules: "Value2", path: "Value2"},
-		&RSRParser{Rules: "~Header3", path: "~Header3"},
+		&RSRParser{Rules: "~Header3", path: "~Header3", rsrRules: make([]*utils.ReSearchReplace, 0)},
 		&RSRParser{Rules: "~Header4:s/a/${1}b/{*duration_seconds&*round:2}",
 			path: "~Header4",
 			rsrRules: []*utils.ReSearchReplace{{
@@ -267,9 +267,9 @@ func TestRSRParsersParseValue(t *testing.T) {
 func TestNewRSRParserMustCompile(t *testing.T) {
 	rsr := NewRSRParserMustCompile("~*req.Account")
 	ePrsr := &RSRParser{
-		Rules: "~*req.Account",
-
-		path: "~*req.Account",
+		Rules:    "~*req.Account",
+		rsrRules: make([]*utils.ReSearchReplace, 0),
+		path:     "~*req.Account",
 	}
 	if !reflect.DeepEqual(rsr, ePrsr) {
 		t.Errorf("Expected %+v received %+v", ePrsr, rsr)

engine/datamanager.go

@@ -714,7 +714,7 @@ func (dm *DataManager) SetFilter(fltr *Filter, withIndex bool) (err error) {
 		return
 	}
 	if withIndex {
-		if err = updateFilterIndex(dm, oldFlt, fltr); err != nil {
+		if err = UpdateFilterIndex(dm, oldFlt, fltr); err != nil {
 			return
 		}
 	}

engine/libindex.go

@@ -448,9 +448,10 @@ func removeIndexFiltersItem(dm *DataManager, idxItmType, tnt, itemID string, fil
 	return
 }
 
-// updateFilterIndex  will update the indexes for the new Filter
+// UpdateFilterIndex  will update the indexes for the new Filter
 // we do not care what is added
-func updateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) {
+// exported for the migrator
+func UpdateFilterIndex(dm *DataManager, oldFlt, newFlt *Filter) (err error) {
 	if oldFlt == nil { // no filter before so no index to update
 		return // nothing to update
 	}

migrator/filters.go

@@ -326,11 +326,6 @@ func (m *Migrator) migrateFilters() (err error) {
 				if fltr, err = m.migrateRequestFilterV4(v4Fltr); err != nil && err != utils.ErrNoMoreData {
 					return
 				}
-
-				// remove the filter to not compile the old rule on set
-				// if err = m.dmOut.DataManager().DataDB().SetFilterDrv(fltr); err != nil {
-				// return
-				// }
 				version = 5
 			}
 			if version == current[utils.RQF] || err == utils.ErrNoMoreData {
@@ -342,7 +337,7 @@ func (m *Migrator) migrateFilters() (err error) {
 			break
 		}
 		if !m.dryRun {
-			if err = m.dmOut.DataManager().SetFilter(fltr, true); err != nil {
+			if err = m.setFilterv5WithoutCompile(fltr); err != nil {
 				return fmt.Errorf("Error: <%s> when setting filter with tenant: <%s> and id: <%s> after migration",
 					err.Error(), fltr.Tenant, fltr.ID)
 			}
@@ -906,3 +901,16 @@ func migrateInlineFilterV4(v4fltIDs []string) (fltrIDs []string, err error) {
 	}
 	return
 }
+
+// setFilterv5WithoutCompile we need a method that get's the filter from DataDB without compiling the filter rules
+func (m *Migrator) setFilterv5WithoutCompile(fltr *engine.Filter) (err error) {
+	var oldFlt *engine.Filter
+	if oldFlt, err = m.dmOut.DataManager().DataDB().GetFilterDrv(fltr.Tenant, fltr.ID); err != nil &&
+		err != utils.ErrNotFound {
+		return
+	}
+	if err = m.dmOut.DataManager().DataDB().SetFilterDrv(fltr); err != nil {
+		return
+	}
+	return engine.UpdateFilterIndex(m.dmOut.DataManager(), oldFlt, fltr)
+}

migrator/filters_it_test.go

@@ -210,6 +210,7 @@ func testFltrITMigrateAndMove(t *testing.T) {
 		} else if vrs[utils.RQF] != 1 {
 			t.Errorf("Unexpected version returned: %d", vrs[utils.RQF])
 		}
+
 		//migrate Filters
 		err, _ = fltrMigrator.Migrate([]string{utils.MetaFilters})
 		if err != nil {
@@ -379,6 +380,7 @@ func testFltrITMigratev2(t *testing.T) {
 	} else if vrs[utils.RQF] != 2 {
 		t.Errorf("Unexpected version returned: %d", vrs[utils.RQF])
 	}
+
 	//migrate Filters
 	err, _ = fltrMigrator.Migrate([]string{utils.MetaFilters})
 	if err != nil {

migrator/stats_it_test.go

@@ -389,8 +389,8 @@ func testStsITMigrateFromv1(t *testing.T) {
 
 	if vrs, err := stsMigrator.dmOut.DataManager().DataDB().GetVersions(utils.StatS); err != nil {
 		t.Errorf("error: <%s> when updating Stats version into dataDB", err.Error())
-	} else if vrs[utils.StatS] != 3 {
-		t.Errorf("Expecting: 3, received: %+v", vrs[utils.StatS])
+	} else if vrs[utils.StatS] != 4 {
+		t.Errorf("Expecting: 4, received: %+v", vrs[utils.StatS])
 	}
 
 	//from V1 to V2