From db1ddeaa43bf79b57e76edb3a1e4775c9980a24a Mon Sep 17 00:00:00 2001 From: Trial97 Date: Thu, 21 Mar 2019 15:41:36 +0200 Subject: [PATCH] Updated and added tests for *remove_session_costs action --- engine/action.go | 5 +- engine/storage_mongo_stordb.go | 6 +- engine/storage_sql.go | 28 +++---- engine/stordb_it_test.go | 66 ++++++++++++++++ utils/apitpdata.go | 37 +++++---- utils/apitpdata_test.go | 135 +++++++++++++++++++++++++++++++++ 6 files changed, 243 insertions(+), 34 deletions(-) diff --git a/engine/action.go b/engine/action.go index af4b0300e..8cd1f4dae 100644 --- a/engine/action.go +++ b/engine/action.go @@ -128,7 +128,7 @@ func getActionFunc(typ string) (actionTypeFunc, bool) { utils.MetaAMQPjsonMap: sendAMQP, utils.MetaAWSjsonMap: sendAWS, utils.MetaSQSjsonMap: sendSQS, - MetaRemoveSessionCosts: cleanSessionCosts, + MetaRemoveSessionCosts: removeSessionCosts, } f, exists := actionFuncMap[typ] return f, exists @@ -963,14 +963,13 @@ func (cdrP *cdrLogProvider) RemoteHost() net.Addr { return utils.LocalAddr() } -func cleanSessionCosts(_ *Account, _ *Action, _ Actions, extraData interface{}) error { // FiltersID;inlineFilter +func removeSessionCosts(_ *Account, _ *Action, _ Actions, extraData interface{}) error { // FiltersID;inlineFilter fltrs, err := utils.IfaceAsString(extraData) if err != nil { return err } tenant := config.CgrConfig().GeneralCfg().DefaultTenant smcFilter := new(utils.SMCostFilter) - smcFilter.Usage = make([]*time.Duration, 2) for _, fltrID := range strings.Split(fltrs, utils.INFIELD_SEP) { fltr, err := dm.GetFilter(tenant, fltrID, true, true, utils.NonTransactional) if err != nil { diff --git a/engine/storage_mongo_stordb.go b/engine/storage_mongo_stordb.go index bb33914b5..c0fdad594 100644 --- a/engine/storage_mongo_stordb.go +++ b/engine/storage_mongo_stordb.go @@ -914,13 +914,13 @@ func (ms *MongoStorage) GetSMCosts(cgrid, runid, originHost, originIDPrefix stri func (ms *MongoStorage) RemoveSMCosts(qryFltr *utils.SMCostFilter) error { filters := bson.M{ - CGRIDLow: bson.M{"$in": qryFltr.CgrIDs, "$nin": qryFltr.NotCgrIDs}, + CGRIDLow: bson.M{"$in": qryFltr.CGRIDs, "$nin": qryFltr.NotCGRIDs}, RunIDLow: bson.M{"$in": qryFltr.RunIDs, "$nin": qryFltr.NotRunIDs}, OriginHostLow: bson.M{"$in": qryFltr.OriginHosts, "$nin": qryFltr.NotOriginHosts}, OriginIDLow: bson.M{"$in": qryFltr.OriginIDs, "$nin": qryFltr.NotOriginIDs}, CostSourceLow: bson.M{"$in": qryFltr.CostSources, "$nin": qryFltr.NotCostSources}, - UsageLow: bson.M{"$gte": qryFltr.Usage[0], "$lt": qryFltr.Usage[1]}, - CreatedAtLow: bson.M{"$gte": qryFltr.CreatedAtStart, "$lt": qryFltr.CreatedAtEnd}, + UsageLow: bson.M{"$gte": qryFltr.Usage.Min, "$lt": qryFltr.Usage.Max}, + CreatedAtLow: bson.M{"$gte": qryFltr.CreatedAt.Begin, "$lt": qryFltr.CreatedAt.End}, } ms.cleanEmptyFilters(filters) return ms.query(func(sctx mongo.SessionContext) (err error) { diff --git a/engine/storage_sql.go b/engine/storage_sql.go index d30f5acdd..22fe9b8e0 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -742,11 +742,11 @@ func (self *SQLStorage) RemoveSMCost(smc *SMCost) error { func (self *SQLStorage) RemoveSMCosts(qryFltr *utils.SMCostFilter) error { q := self.db.Table(utils.SessionCostsTBL).Select("*") // Add filters, use in to replace the high number of ORs - if len(qryFltr.CgrIDs) != 0 { - q = q.Where("cgrid in (?)", qryFltr.CgrIDs) + if len(qryFltr.CGRIDs) != 0 { + q = q.Where("cgrid in (?)", qryFltr.CGRIDs) } - if len(qryFltr.NotCgrIDs) != 0 { - q = q.Where("cgrid not in (?)", qryFltr.NotCgrIDs) + if len(qryFltr.NotCGRIDs) != 0 { + q = q.Where("cgrid not in (?)", qryFltr.NotCGRIDs) } if len(qryFltr.RunIDs) != 0 { q = q.Where("run_id in (?)", qryFltr.RunIDs) @@ -772,24 +772,24 @@ func (self *SQLStorage) RemoveSMCosts(qryFltr *utils.SMCostFilter) error { if len(qryFltr.NotCostSources) != 0 { q = q.Where("costsource not in (?)", qryFltr.NotCostSources) } - if qryFltr.CreatedAtStart != nil { - q = q.Where("created_at >= ?", qryFltr.CreatedAtStart) + if qryFltr.CreatedAt.Begin != nil { + q = q.Where("created_at >= ?", qryFltr.CreatedAt.Begin) } - if qryFltr.CreatedAtEnd != nil { - q = q.Where("created_at < ?", qryFltr.CreatedAtEnd) + if qryFltr.CreatedAt.End != nil { + q = q.Where("created_at < ?", qryFltr.CreatedAt.End) } - if qryFltr.Usage[0] != nil { + if qryFltr.Usage.Min != nil { if self.db.Dialect().GetName() == utils.MYSQL { // MySQL needs escaping for usage - q = q.Where("`usage` >= ?", qryFltr.Usage[0].Nanoseconds()) + q = q.Where("`usage` >= ?", qryFltr.Usage.Min.Nanoseconds()) } else { - q = q.Where("usage >= ?", qryFltr.Usage[0].Nanoseconds()) + q = q.Where("usage >= ?", qryFltr.Usage.Min.Nanoseconds()) } } - if qryFltr.Usage[1] != nil { + if qryFltr.Usage.Max != nil { if self.db.Dialect().GetName() == utils.MYSQL { // MySQL needs escaping for usage - q = q.Where("`usage` < ?", qryFltr.Usage[1].Nanoseconds()) + q = q.Where("`usage` < ?", qryFltr.Usage.Max.Nanoseconds()) } else { - q = q.Where("usage < ?", qryFltr.Usage[1].Nanoseconds()) + q = q.Where("usage < ?", qryFltr.Usage.Max.Nanoseconds()) } } if err := q.Delete(nil).Error; err != nil { diff --git a/engine/stordb_it_test.go b/engine/stordb_it_test.go index 129a60b00..588a6aade 100644 --- a/engine/stordb_it_test.go +++ b/engine/stordb_it_test.go @@ -57,6 +57,7 @@ var sTestsStorDBit = []func(t *testing.T){ testStorDBitCRUDTpStats, testStorDBitCRUDCDRs, testStorDBitCRUDSMCosts, + testStorDBitCRUDSMCosts2, } func TestStorDBitMySQL(t *testing.T) { @@ -1369,6 +1370,71 @@ func testStorDBitCRUDSMCosts(t *testing.T) { } } +func testStorDBitCRUDSMCosts2(t *testing.T) { + // READ + if _, err := storDB.GetSMCosts("", "", "", ""); err != utils.ErrNotFound { + t.Error(err) + } + // WRITE + var snd = []*SMCost{ + { + CGRID: "CGRID1", + RunID: "11", + OriginHost: "host22", + OriginID: "O1", + CostDetails: NewBareEventCost(), + }, + { + CGRID: "CGRID2", + RunID: "12", + OriginHost: "host22", + OriginID: "O2", + CostDetails: NewBareEventCost(), + }, + { + CGRID: "CGRID3", + RunID: "13", + OriginHost: "host23", + OriginID: "O3", + CostDetails: NewBareEventCost(), + }, + } + for _, smc := range snd { + if err := storDB.SetSMCost(smc); err != nil { + t.Error(err) + } + } + // READ + if rcv, err := storDB.GetSMCosts("", "", "host22", ""); err != nil { + t.Fatal(err) + } else if len(rcv) != 2 { + t.Errorf("Expected 2 results received %v ", len(rcv)) + } + // REMOVE + if err := storDB.RemoveSMCosts(&utils.SMCostFilter{ + RunIDs: []string{"12", "13"}, + NotRunIDs: []string{"11"}, + OriginHosts: []string{"host22", "host23"}, + NotOriginHosts: []string{"host21"}, + }); err != nil { + t.Error(err) + } + // READ + if rcv, err := storDB.GetSMCosts("", "", "", ""); err != nil { + t.Error(err) + } else if len(rcv) != 1 { + t.Errorf("Expected 1 result received %v ", len(rcv)) + } + // REMOVE + if err := storDB.RemoveSMCosts(&utils.SMCostFilter{}); err != nil { + t.Error(err) + } + // READ + if _, err := storDB.GetSMCosts("", "", "", ""); err != utils.ErrNotFound { + t.Error(err) + } +} + func testStorDBitFlush(t *testing.T) { if err := storDB.Flush(path.Join(cfg.DataFolderPath, "storage", cfg.StorDbCfg().StorDBType)); err != nil { t.Error(err) diff --git a/utils/apitpdata.go b/utils/apitpdata.go index fa556ec57..d80223f06 100755 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -1203,9 +1203,19 @@ type TPDispatcherProfile struct { Conns []*TPDispatcherConns } +type UsageInterval struct { + Min *time.Duration + Max *time.Duration +} + +type TimeInterval struct { + Begin *time.Time + End *time.Time +} + type SMCostFilter struct { //id cu litere mare - CgrIDs []string - NotCgrIDs []string + CGRIDs []string + NotCGRIDs []string RunIDs []string NotRunIDs []string OriginHosts []string @@ -1214,9 +1224,8 @@ type SMCostFilter struct { //id cu litere mare NotOriginIDs []string CostSources []string NotCostSources []string - Usage []*time.Duration // slice min=Usage[0]&max=Usage[1] - CreatedAtStart *time.Time // Start of interval, bigger or equal than configured - CreatedAtEnd *time.Time // End interval, smaller than + Usage UsageInterval + CreatedAt TimeInterval } func AppendToSMCostFilter(smcFilter *SMCostFilter, fieldType, fieldName string, values []string, timezone string) (smcf *SMCostFilter, err error) { @@ -1230,9 +1239,9 @@ func AppendToSMCostFilter(smcFilter *SMCostFilter, fieldType, fieldName string, case DynamicDataPrefix + CGRID: switch fieldType { case MetaString: - smcFilter.CgrIDs = append(smcFilter.CgrIDs, values...) + smcFilter.CGRIDs = append(smcFilter.CGRIDs, values...) case MetaNotString: - smcFilter.NotCgrIDs = append(smcFilter.NotCgrIDs, values...) + smcFilter.NotCGRIDs = append(smcFilter.NotCGRIDs, values...) default: err = fmt.Errorf("FilterType: %q not supported for FieldName: %q", fieldType, fieldName) } @@ -1268,7 +1277,7 @@ func AppendToSMCostFilter(smcFilter *SMCostFilter, fieldType, fieldName string, case MetaString: smcFilter.CostSources = append(smcFilter.CostSources, values...) case MetaNotString: - smcFilter.CostSources = append(smcFilter.NotCostSources, values...) + smcFilter.NotCostSources = append(smcFilter.NotCostSources, values...) default: err = fmt.Errorf("FilterType: %q not supported for FieldName: %q", fieldType, fieldName) } @@ -1278,18 +1287,18 @@ func AppendToSMCostFilter(smcFilter *SMCostFilter, fieldType, fieldName string, var minUsage time.Duration minUsage, err = ParseDurationWithNanosecs(values[0]) if err != nil { - err = fmt.Errorf("Error when converting field: %q value: %q in float ", fieldType, fieldName) + err = fmt.Errorf("Error when converting field: %q value: %q in time.Duration ", fieldType, fieldName) break } - smcFilter.Usage[0] = &minUsage + smcFilter.Usage.Min = &minUsage case MetaLessThan: var maxUsage time.Duration maxUsage, err = ParseDurationWithNanosecs(values[0]) if err != nil { - err = fmt.Errorf("Error when converting field: %q value: %q in float ", fieldType, fieldName) + err = fmt.Errorf("Error when converting field: %q value: %q in time.Duration ", fieldType, fieldName) break } - smcFilter.Usage[1] = &maxUsage + smcFilter.Usage.Max = &maxUsage default: err = fmt.Errorf("FilterType: %q not supported for FieldName: %q", fieldType, fieldName) } @@ -1303,7 +1312,7 @@ func AppendToSMCostFilter(smcFilter *SMCostFilter, fieldType, fieldName string, break } if !start.IsZero() { - smcFilter.CreatedAtStart = &start + smcFilter.CreatedAt.Begin = &start } case MetaLessThan: var end time.Time @@ -1313,7 +1322,7 @@ func AppendToSMCostFilter(smcFilter *SMCostFilter, fieldType, fieldName string, break } if !end.IsZero() { - smcFilter.CreatedAtEnd = &end + smcFilter.CreatedAt.End = &end } default: err = fmt.Errorf("FilterType: %q not supported for FieldName: %q", fieldType, fieldName) diff --git a/utils/apitpdata_test.go b/utils/apitpdata_test.go index 54fc1a79c..f4a2f4396 100644 --- a/utils/apitpdata_test.go +++ b/utils/apitpdata_test.go @@ -20,6 +20,7 @@ package utils import ( "reflect" "testing" + "time" ) func TestNewDTCSFromRPKey(t *testing.T) { @@ -59,3 +60,137 @@ func TestPaginatorPaginateStringSlice(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", eOut, rcv) } } + +func TestAppendToSMCostFilter(t *testing.T) { + var err error + smfltr := new(SMCostFilter) + expected := &SMCostFilter{ + CGRIDs: []string{"CGRID1", "CGRID2"}, + } + if smfltr, err = AppendToSMCostFilter(smfltr, "*string", DynamicDataPrefix+CGRID, []string{"CGRID1", "CGRID2"}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + expected.NotCGRIDs = []string{"CGRID3", "CGRID4"} + if smfltr, err = AppendToSMCostFilter(smfltr, "*notstring", DynamicDataPrefix+CGRID, []string{"CGRID3", "CGRID4"}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + + expected.RunIDs = []string{"RunID1", "RunID2"} + if smfltr, err = AppendToSMCostFilter(smfltr, "*string", DynamicDataPrefix+RunID, []string{"RunID1", "RunID2"}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + expected.NotRunIDs = []string{"RunID3", "RunID4"} + if smfltr, err = AppendToSMCostFilter(smfltr, "*notstring", DynamicDataPrefix+RunID, []string{"RunID3", "RunID4"}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + + expected.OriginHosts = []string{"OriginHost1", "OriginHost2"} + if smfltr, err = AppendToSMCostFilter(smfltr, "*string", DynamicDataPrefix+OriginHost, []string{"OriginHost1", "OriginHost2"}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + expected.NotOriginHosts = []string{"OriginHost3", "OriginHost4"} + if smfltr, err = AppendToSMCostFilter(smfltr, "*notstring", DynamicDataPrefix+OriginHost, []string{"OriginHost3", "OriginHost4"}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + + expected.OriginIDs = []string{"OriginID1", "OriginID2"} + if smfltr, err = AppendToSMCostFilter(smfltr, "*string", DynamicDataPrefix+OriginID, []string{"OriginID1", "OriginID2"}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + expected.NotOriginIDs = []string{"OriginID3", "OriginID4"} + if smfltr, err = AppendToSMCostFilter(smfltr, "*notstring", DynamicDataPrefix+OriginID, []string{"OriginID3", "OriginID4"}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + + expected.CostSources = []string{"CostSource1", "CostSource2"} + if smfltr, err = AppendToSMCostFilter(smfltr, "*string", DynamicDataPrefix+CostSource, []string{"CostSource1", "CostSource2"}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + expected.NotCostSources = []string{"CostSource3", "CostSource4"} + if smfltr, err = AppendToSMCostFilter(smfltr, "*notstring", DynamicDataPrefix+CostSource, []string{"CostSource3", "CostSource4"}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + + if smfltr, err = AppendToSMCostFilter(smfltr, "*prefix", DynamicDataPrefix+CGRID, []string{"CGRID1", "CGRID2"}, ""); err == nil || err.Error() != "FilterType: \"*prefix\" not supported for FieldName: \"~CGRID\"" { + t.Errorf("Expected error: FilterType: \"*prefix\" not supported for FieldName: \"~CGRID\" ,received %v", err) + } + if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + if smfltr, err = AppendToSMCostFilter(smfltr, "*string", CGRID, []string{"CGRID1", "CGRID2"}, ""); err == nil || err.Error() != "FieldName: \"CGRID\" not supported" { + t.Errorf("Expected error: FieldName: \"CGRID\" not supported ,received %v", err) + } + if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + if smfltr, err = AppendToSMCostFilter(smfltr, "*prefix", CGRID, []string{"CGRID1", "CGRID2"}, ""); err == nil || err.Error() != "FieldName: \"CGRID\" not supported" { + t.Errorf("Expected error: FieldName: \"CGRID\" not supported ,received %v", err) + } + if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + + } + expected.Usage.Min = DurationPointer(time.Second) + if smfltr, err = AppendToSMCostFilter(smfltr, "*gte", DynamicDataPrefix+Usage, []string{"1s", "2s"}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + expected.Usage.Max = DurationPointer(3 * time.Second) + if smfltr, err = AppendToSMCostFilter(smfltr, "*lt", DynamicDataPrefix+Usage, []string{"3s", "4s"}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + + if smfltr, err = AppendToSMCostFilter(smfltr, "*gte", DynamicDataPrefix+Usage, []string{"one second"}, ""); err == nil || err.Error() != "Error when converting field: \"*gte\" value: \"~Usage\" in time.Duration " { + t.Errorf("Expected error: Error when converting field: \"*gte\" value: \"~Usage\" in time.Duration ,received %v", err) + } + if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + + now := time.Now().UTC().Round(time.Second) + strNow := now.Format("2006-01-02T15:04:05") + + expected.CreatedAt.Begin = &now + if smfltr, err = AppendToSMCostFilter(smfltr, "*gte", DynamicDataPrefix+CreatedAt, []string{strNow}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + + expected.CreatedAt.End = &now + if smfltr, err = AppendToSMCostFilter(smfltr, "*lt", DynamicDataPrefix+CreatedAt, []string{strNow}, ""); err != nil { + t.Error(err) + } else if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } + + if smfltr, err = AppendToSMCostFilter(smfltr, "*gte", DynamicDataPrefix+CreatedAt, []string{time.Now().String()}, ""); err == nil || err.Error() != "Error when converting field: \"*gte\" value: \"~CreatedAt\" in time.Time " { + t.Errorf("Expected error: Error when converting field: \"*gte\" value: \"~CreatedAt\" in time.Time ,received %v", err) + } + if !reflect.DeepEqual(smfltr, expected) { + t.Errorf("Expected: %s ,received: %s ", ToJSON(expected), ToJSON(smfltr)) + } +}