From c85e0389bcc82309c7157ad2e3be114c50cb9d35 Mon Sep 17 00:00:00 2001 From: Trial97 Date: Thu, 13 Aug 2020 17:28:34 +0300 Subject: [PATCH] Updated GetCDRs for internalDB --- engine/storage_internal_stordb.go | 77 +++++++++++++++++++------------ engine/z_storage_cdrs_it_test.go | 55 ++++++++++++++++++---- 2 files changed, 93 insertions(+), 39 deletions(-) diff --git a/engine/storage_internal_stordb.go b/engine/storage_internal_stordb.go index 030957029..493dc1bc9 100644 --- a/engine/storage_internal_stordb.go +++ b/engine/storage_internal_stordb.go @@ -30,7 +30,7 @@ import ( // GetTpIds implements LoadReader interface func (iDB *InternalDB) GetTpIds(colName string) (ids []string, err error) { tpIDs := make(utils.StringSet) - if colName == "" { // if colName is empty we need to parse all partitions + if colName == utils.EmptyString { // if colName is empty we need to parse all partitions for _, conNm := range utils.CacheStorDBPartitions { // iterate through all columns for _, key := range Cache.GetItemIDs(conNm, utils.EmptyString) { tpIDs.Add(strings.Split(key, utils.InInFieldSep)[0]) @@ -1622,37 +1622,45 @@ func (iDB *InternalDB) GetCDRs(filter *utils.CDRsFilter, remove bool) (cdrs []*C } } - if filter.Paginator.Offset != nil && - paginatorOffsetCounter <= *filter.Paginator.Offset { - paginatorOffsetCounter++ - continue - } - if filter.Paginator.Limit != nil && - len(cdrs) >= *filter.Paginator.Limit { - break + if filter.OrderBy == utils.EmptyString { // if do not have to order exit early + if filter.Paginator.Offset != nil && + paginatorOffsetCounter <= *filter.Paginator.Offset { + paginatorOffsetCounter++ + continue + } + if filter.Paginator.Limit != nil && + len(cdrs) >= *filter.Paginator.Limit { + break + } } //pass all filters and append to slice cdrs = append(cdrs, cdr) } - if filter.Count { - return nil, int64(len(cdrs)), nil - } - if remove { - for _, cdr := range cdrs { - Cache.RemoveWithoutReplicate(utils.CacheCDRsTBL, utils.ConcatenatedKey(cdr.CGRID, cdr.RunID, cdr.OriginID), - cacheCommit(utils.NonTransactional), utils.NonTransactional) + if filter.OrderBy != utils.EmptyString && + filter.Paginator.Offset != nil && + len(cdrs) < *filter.Paginator.Offset { // if we have offset populated but not enough cdrs return + if filter.Count { + return nil, 0, nil } - return nil, 0, nil - } - if len(cdrs) == 0 { return nil, 0, utils.ErrNotFound } - if filter.OrderBy != "" { - separateVals := strings.Split(filter.OrderBy, utils.INFIELD_SEP) - ascendent := true - if len(separateVals) == 2 && separateVals[1] == "desc" { - ascendent = false + if filter.Count { + if filter.Paginator.Limit != nil && + len(cdrs) >= *filter.Paginator.Limit { + return nil, int64(*filter.Paginator.Limit), nil } + return nil, int64(len(cdrs)), nil + } + if len(cdrs) == 0 { + if remove { + return nil, 0, nil + } + return nil, 0, utils.ErrNotFound + } + + if filter.OrderBy != utils.EmptyString { + separateVals := strings.Split(filter.OrderBy, utils.INFIELD_SEP) + ascendent := !(len(separateVals) == 2 && separateVals[1] == "desc") switch separateVals[0] { case utils.OrderID: if ascendent { @@ -1664,7 +1672,6 @@ func (iDB *InternalDB) GetCDRs(filter *utils.CDRsFilter, remove bool) (cdrs []*C return cdrs[i].OrderID > cdrs[j].OrderID }) } - case utils.AnswerTime: if ascendent { sort.Slice(cdrs, func(i, j int) bool { @@ -1675,7 +1682,6 @@ func (iDB *InternalDB) GetCDRs(filter *utils.CDRsFilter, remove bool) (cdrs []*C return cdrs[i].AnswerTime.After(cdrs[j].AnswerTime) }) } - case utils.SetupTime: if ascendent { sort.Slice(cdrs, func(i, j int) bool { @@ -1686,7 +1692,6 @@ func (iDB *InternalDB) GetCDRs(filter *utils.CDRsFilter, remove bool) (cdrs []*C return cdrs[i].SetupTime.After(cdrs[j].SetupTime) }) } - case utils.Usage: if ascendent { sort.Slice(cdrs, func(i, j int) bool { @@ -1697,7 +1702,6 @@ func (iDB *InternalDB) GetCDRs(filter *utils.CDRsFilter, remove bool) (cdrs []*C return cdrs[i].Usage > cdrs[j].Usage }) } - case utils.Cost: if ascendent { sort.Slice(cdrs, func(i, j int) bool { @@ -1708,10 +1712,25 @@ func (iDB *InternalDB) GetCDRs(filter *utils.CDRsFilter, remove bool) (cdrs []*C return cdrs[i].Cost > cdrs[j].Cost }) } - default: return nil, 0, fmt.Errorf("Invalid value : %s", separateVals[0]) } + + if filter.Paginator.Offset != nil { + cdrs = cdrs[*filter.Paginator.Offset:] + } + if filter.Paginator.Limit != nil { + if len(cdrs) > *filter.Paginator.Limit { + cdrs = cdrs[:*filter.Paginator.Limit] + } + } + } + if remove { + for _, cdr := range cdrs { + Cache.RemoveWithoutReplicate(utils.CacheCDRsTBL, utils.ConcatenatedKey(cdr.CGRID, cdr.RunID, cdr.OriginID), + cacheCommit(utils.NonTransactional), utils.NonTransactional) + } + return nil, 0, nil } return } diff --git a/engine/z_storage_cdrs_it_test.go b/engine/z_storage_cdrs_it_test.go index 5a322b11a..d543b6ec4 100644 --- a/engine/z_storage_cdrs_it_test.go +++ b/engine/z_storage_cdrs_it_test.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "path" + "reflect" "strconv" "testing" "time" @@ -266,7 +267,7 @@ func testGetCDRs(cfg *config.CGRConfig) error { Destination: "1002", SetupTime: time.Date(2015, 12, 12, 14, 52, 0, 0, time.UTC), AnswerTime: time.Date(2015, 12, 12, 14, 52, 20, 0, time.UTC), - Usage: time.Duration(35) * time.Second, + Usage: 35 * time.Second, ExtraFields: map[string]string{"ExtraHeader1": "ExtraVal1", "ExtraHeader2": "ExtraVal2"}, CostSource: "", Cost: -1, @@ -286,7 +287,7 @@ func testGetCDRs(cfg *config.CGRConfig) error { Destination: "1002", SetupTime: time.Date(2015, 12, 12, 14, 52, 0, 0, time.UTC), AnswerTime: time.Date(2015, 12, 12, 14, 52, 20, 0, time.UTC), - Usage: time.Duration(35) * time.Second, + Usage: 35 * time.Second, ExtraFields: map[string]string{"ExtraHeader1": "ExtraVal1", "ExtraHeader2": "ExtraVal2"}, CostSource: "testGetCDRs", Cost: 0.17, @@ -306,7 +307,7 @@ func testGetCDRs(cfg *config.CGRConfig) error { Destination: "1002", SetupTime: time.Date(2015, 12, 12, 14, 52, 0, 0, time.UTC), AnswerTime: time.Date(2015, 12, 12, 14, 52, 20, 0, time.UTC), - Usage: time.Duration(35) * time.Second, + Usage: 35 * time.Second, ExtraFields: map[string]string{"ExtraHeader1": "ExtraVal1", "ExtraHeader2": "ExtraVal2"}, CostSource: "testGetCDRs", Cost: 0.17, @@ -326,7 +327,7 @@ func testGetCDRs(cfg *config.CGRConfig) error { Destination: "1007", SetupTime: time.Date(2015, 12, 29, 12, 58, 0, 0, time.UTC), AnswerTime: time.Date(2015, 12, 29, 12, 59, 0, 0, time.UTC), - Usage: time.Duration(0) * time.Second, + Usage: 0, ExtraFields: map[string]string{"ExtraHeader1": "ExtraVal1", "ExtraHeader2": "ExtraVal2"}, CostSource: "rater1", Cost: 0, @@ -346,7 +347,7 @@ func testGetCDRs(cfg *config.CGRConfig) error { Destination: "1003", SetupTime: time.Date(2015, 12, 28, 12, 58, 0, 0, time.UTC), AnswerTime: time.Date(2015, 12, 28, 12, 58, 30, 0, time.UTC), - Usage: time.Duration(125) * time.Second, + Usage: 125 * time.Second, ExtraFields: map[string]string{}, CostSource: "", Cost: -1, @@ -366,7 +367,7 @@ func testGetCDRs(cfg *config.CGRConfig) error { Destination: "1003", SetupTime: time.Date(2015, 12, 28, 12, 58, 0, 0, time.UTC), AnswerTime: time.Date(2015, 12, 28, 12, 58, 30, 0, time.UTC), - Usage: time.Duration(125) * time.Second, + Usage: 125 * time.Second, ExtraFields: map[string]string{}, CostSource: "testSetCDRs", Cost: -1, @@ -387,7 +388,7 @@ func testGetCDRs(cfg *config.CGRConfig) error { Destination: "1007", SetupTime: time.Date(2015, 12, 14, 14, 52, 0, 0, time.UTC), AnswerTime: time.Date(2015, 12, 12, 14, 52, 20, 0, time.UTC), - Usage: time.Duration(64) * time.Second, + Usage: 64 * time.Second, ExtraFields: map[string]string{"ExtraHeader3": "ExtraVal3"}, CostSource: "", Cost: -1, @@ -407,7 +408,7 @@ func testGetCDRs(cfg *config.CGRConfig) error { Destination: "1007", SetupTime: time.Date(2015, 12, 14, 14, 52, 0, 0, time.UTC), AnswerTime: time.Date(2015, 12, 12, 14, 52, 20, 0, time.UTC), - Usage: time.Duration(64) * time.Second, + Usage: 64 * time.Second, ExtraFields: map[string]string{"ExtraHeader3": "ExtraVal3"}, CostSource: "testSetCDRs", Cost: 1.205, @@ -427,7 +428,7 @@ func testGetCDRs(cfg *config.CGRConfig) error { Destination: "1002", SetupTime: time.Date(2015, 12, 15, 18, 22, 0, 0, time.UTC), AnswerTime: time.Date(2015, 12, 15, 18, 22, 0, 0, time.UTC), - Usage: time.Duration(1) * time.Second, + Usage: time.Second, ExtraFields: map[string]string{"Service-Context-Id": "voice@huawei.com"}, CostSource: "", Cost: -1, @@ -447,7 +448,7 @@ func testGetCDRs(cfg *config.CGRConfig) error { Destination: "1002", SetupTime: time.Date(2015, 12, 15, 18, 22, 0, 0, time.UTC), AnswerTime: time.Date(2015, 12, 15, 18, 22, 0, 0, time.UTC), - Usage: time.Duration(1) * time.Second, + Usage: time.Second, ExtraFields: map[string]string{"Service-Context-Id": "voice2@huawei.com"}, CostSource: "rater", Cost: 0.15, @@ -811,5 +812,39 @@ func testGetCDRs(cfg *config.CGRConfig) error { } } } + + // Limit 5 with filter + if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Accounts: []string{"1001"}, Paginator: utils.Paginator{Limit: utils.IntPointer(2), Offset: utils.IntPointer(0)}}, false); err != nil { + return fmt.Errorf("testGetCDRs #99 err: %v", err) + } else if len(CDRs) != 2 { + return fmt.Errorf("testGetCDRs #100, unexpected number of CDRs returned: %+v", len(CDRs)) + } + + //Filter by OrderID with paginator + if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{OrderBy: "OrderID", Paginator: utils.Paginator{Limit: utils.IntPointer(3)}}, false); err != nil { + return fmt.Errorf("testGetCDRs #101, err: %v", err) + } else if !reflect.DeepEqual(cdrs[:3], CDRs) { + return fmt.Errorf("testGetCDRs #102 Expected %+v received %+v \n", cdrs, CDRs) + } + + if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{OrderBy: "OrderID", Paginator: utils.Paginator{Limit: utils.IntPointer(5)}}, false); err != nil { + return fmt.Errorf("testGetCDRs #103, err: %v", err) + } else if !reflect.DeepEqual(cdrs[:5], CDRs) { + return fmt.Errorf("testGetCDRs #104 Expected %+v received %+v \n", cdrs, CDRs) + } + + if CDRs, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{OrderBy: "OrderID", Paginator: utils.Paginator{Limit: utils.IntPointer(3), Offset: utils.IntPointer(2)}}, false); err != nil { + return fmt.Errorf("testGetCDRs #103, err: %v", err) + } else if !reflect.DeepEqual(cdrs[2:5], CDRs) { + return fmt.Errorf("testGetCDRs #104 Expected %+v received %+v \n", utils.ToJSON(cdrs[2:5]), utils.ToJSON(CDRs)) + } + + if _, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{Paginator: utils.Paginator{Limit: utils.IntPointer(3), Offset: utils.IntPointer(20)}}, false); err != utils.ErrNotFound { + return fmt.Errorf("testGetCDRs #105, err: %v", err) + } + + if _, _, err := cdrStorage.GetCDRs(&utils.CDRsFilter{OrderBy: "OrderID", Paginator: utils.Paginator{Limit: utils.IntPointer(3), Offset: utils.IntPointer(20)}}, false); err != utils.ErrNotFound { + return fmt.Errorf("testGetCDRs #105, err: %v", err) + } return nil }