From 716ba68f425cb1305866f7af400f7f0959a08201 Mon Sep 17 00:00:00 2001 From: TeoV Date: Tue, 10 Oct 2017 18:16:16 +0300 Subject: [PATCH 01/10] Updated filters_it_test.go in apier/v1 --- apier/v1/filters_it_test.go | 166 ++++++++++++++++++------------------ 1 file changed, 82 insertions(+), 84 deletions(-) diff --git a/apier/v1/filters_it_test.go b/apier/v1/filters_it_test.go index 6c7e3c8dd..8f015e2db 100644 --- a/apier/v1/filters_it_test.go +++ b/apier/v1/filters_it_test.go @@ -20,7 +20,6 @@ along with this program. If not, see package v1 -/* import ( "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" @@ -33,169 +32,168 @@ import ( ) var ( - filterProfileCfgPath string - filterProfileCfg *config.CGRConfig - filterProfileRPC *rpc.Client - filterProfileDataDir = "/usr/share/cgrates" - filterProfile *engine.FilterProfile - filterProfileDelay int - filterProfileConfigDIR string //run tests for specific configuration + filterCfgPath string + filterCfg *config.CGRConfig + filterRPC *rpc.Client + filterDataDir = "/usr/share/cgrates" + filter *engine.Filter + filterDelay int + filterConfigDIR string //run tests for specific configuration ) -var sTestsFilterProfile = []func(t *testing.T){ - testFilterProfileInitCfg, - testFilterProfileResetDataDB, - testFilterProfileStartEngine, - testFilterProfileRpcConn, - testFilterProfileGetFilterProfileBeforeSet, - testFilterProfileSetFilterProfile, - testFilterProfileGetFilterProfileAfterSet, - testFilterProfileUpdateFilterProfile, - testFilterProfileGetFilterProfileAfterUpdate, - testFilterProfileRemFilterProfile, - testFilterProfileGetFilterProfileAfterRemove, - testFilterProfileKillEngine, +var sTestsFilter = []func(t *testing.T){ + testFilterInitCfg, + testFilterResetDataDB, + testFilterStartEngine, + testFilterRpcConn, + testFilterGetFilterBeforeSet, + testFilterSetFilter, + testFilterGetFilterAfterSet, + testFilterUpdateFilter, + testFilterGetFilterAfterUpdate, + testFilterRemFilter, + testFilterGetFilterAfterRemove, + testFilterKillEngine, } //Test start here -func TestFilterProfileITMySql(t *testing.T) { - filterProfileConfigDIR = "tutmysql" - for _, stest := range sTestsFilterProfile { - t.Run(filterProfileConfigDIR, stest) +func TestFilterITMySql(t *testing.T) { + filterConfigDIR = "tutmysql" + for _, stest := range sTestsFilter { + t.Run(filterConfigDIR, stest) } } -func TestFilterProfileITMongo(t *testing.T) { - filterProfileConfigDIR = "tutmongo" - for _, stest := range sTestsFilterProfile { - t.Run(filterProfileConfigDIR, stest) +func TestFilterITMongo(t *testing.T) { + filterConfigDIR = "tutmongo" + for _, stest := range sTestsFilter { + t.Run(filterConfigDIR, stest) } } -func TestFilterProfileITPG(t *testing.T) { - filterProfileConfigDIR = "tutpostgres" - for _, stest := range sTestsFilterProfile { - t.Run(filterProfileConfigDIR, stest) +func TestFilterITPG(t *testing.T) { + filterConfigDIR = "tutpostgres" + for _, stest := range sTestsFilter { + t.Run(filterConfigDIR, stest) } } -func testFilterProfileInitCfg(t *testing.T) { +func testFilterInitCfg(t *testing.T) { var err error - filterProfileCfgPath = path.Join(filterProfileDataDir, "conf", "samples", filterProfileConfigDIR) - filterProfileCfg, err = config.NewCGRConfigFromFolder(filterProfileCfgPath) + filterCfgPath = path.Join(filterDataDir, "conf", "samples", filterConfigDIR) + filterCfg, err = config.NewCGRConfigFromFolder(filterCfgPath) if err != nil { t.Error(err) } - filterProfileCfg.DataFolderPath = filterProfileDataDir // Share DataFolderPath through config towards StoreDb for Flush() - config.SetCgrConfig(filterProfileCfg) - switch filterProfileConfigDIR { + filterCfg.DataFolderPath = filterDataDir // Share DataFolderPath through config towards StoreDb for Flush() + config.SetCgrConfig(filterCfg) + switch filterConfigDIR { case "tutmongo": // Mongo needs more time to reset db, need to investigate - filterProfileDelay = 2000 + filterDelay = 2000 default: - filterProfileDelay = 1000 + filterDelay = 1000 } } // Wipe out the cdr database -func testFilterProfileResetDataDB(t *testing.T) { - if err := engine.InitDataDb(filterProfileCfg); err != nil { +func testFilterResetDataDB(t *testing.T) { + if err := engine.InitDataDb(filterCfg); err != nil { t.Fatal(err) } } // Start CGR Engine -func testFilterProfileStartEngine(t *testing.T) { - if _, err := engine.StopStartEngine(filterProfileCfgPath, filterProfileDelay); err != nil { +func testFilterStartEngine(t *testing.T) { + if _, err := engine.StopStartEngine(filterCfgPath, filterDelay); err != nil { t.Fatal(err) } } // Connect rpc client to rater -func testFilterProfileRpcConn(t *testing.T) { +func testFilterRpcConn(t *testing.T) { var err error - filterProfileRPC, err = jsonrpc.Dial("tcp", filterProfileCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed + filterRPC, err = jsonrpc.Dial("tcp", filterCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed if err != nil { t.Fatal(err) } } -func testFilterProfileGetFilterProfileBeforeSet(t *testing.T) { - var reply *engine.FilterProfile - if err := filterProfileRPC.Call("ApierV1.GetFilterProfile", &utils.TenantID{Tenant: "cgrates.org", ID: "Filter1"}, &reply); err == nil || err.Error() != utils.ErrNotFound.Error() { +func testFilterGetFilterBeforeSet(t *testing.T) { + var reply *engine.Filter + if err := filterRPC.Call("ApierV1.GetFilter", &utils.TenantID{Tenant: "cgrates.org", ID: "Filter1"}, &reply); err == nil || err.Error() != utils.ErrNotFound.Error() { t.Error(err) } } -func testFilterProfileSetFilterProfile(t *testing.T) { - filterProfile = &engine.FilterProfile{ - Tenant: "cgrates.org", - ID: "Filter1", - FilterType: "*string_prefix", - FilterFieldName: "Account", - FilterFieldValues: []string{"10", "20"}, +func testFilterSetFilter(t *testing.T) { + filter = &engine.Filter{ + Tenant: "cgrates.org", + ID: "Filter1", + Type: "*string_prefix", + FieldName: "Account", + Values: []string{"10", "20"}, } var result string - if err := filterProfileRPC.Call("ApierV1.SetFilterProfile", filterProfile, &result); err != nil { + if err := filterRPC.Call("ApierV1.SetFilter", filter, &result); err != nil { t.Error(err) } else if result != utils.OK { t.Error("Unexpected reply returned", result) } } -func testFilterProfileGetFilterProfileAfterSet(t *testing.T) { - var reply *engine.FilterProfile - if err := filterProfileRPC.Call("ApierV1.GetFilterProfile", &utils.TenantID{Tenant: "cgrates.org", ID: "Filter1"}, &reply); err != nil { +func testFilterGetFilterAfterSet(t *testing.T) { + var reply *engine.Filter + if err := filterRPC.Call("ApierV1.GetFilter", &utils.TenantID{Tenant: "cgrates.org", ID: "Filter1"}, &reply); err != nil { t.Error(err) - } else if !reflect.DeepEqual(filterProfile, reply) { - t.Errorf("Expecting : %+v, received: %+v", filterProfile, reply) + } else if !reflect.DeepEqual(filter, reply) { + t.Errorf("Expecting : %+v, received: %+v", filter, reply) } } -func testFilterProfileUpdateFilterProfile(t *testing.T) { - filterProfile = &engine.FilterProfile{ - Tenant: "cgrates.org", - ID: "Filter1", - FilterType: "*string_prefix", - FilterFieldName: "Destination", - FilterFieldValues: []string{"1001", "1002"}, +func testFilterUpdateFilter(t *testing.T) { + filter = &engine.Filter{ + Tenant: "cgrates.org", + ID: "Filter1", + Type: "*string_prefix", + FieldName: "Destination", + Values: []string{"1001", "1002"}, } var result string - if err := filterProfileRPC.Call("ApierV1.SetFilterProfile", filterProfile, &result); err != nil { + if err := filterRPC.Call("ApierV1.SetFilter", filter, &result); err != nil { t.Error(err) } else if result != utils.OK { t.Error("Unexpected reply returned", result) } } -func testFilterProfileGetFilterProfileAfterUpdate(t *testing.T) { - var reply *engine.FilterProfile - if err := filterProfileRPC.Call("ApierV1.GetFilterProfile", &utils.TenantID{Tenant: "cgrates.org", ID: "Filter1"}, &reply); err != nil { +func testFilterGetFilterAfterUpdate(t *testing.T) { + var reply *engine.Filter + if err := filterRPC.Call("ApierV1.GetFilter", &utils.TenantID{Tenant: "cgrates.org", ID: "Filter1"}, &reply); err != nil { t.Error(err) - } else if !reflect.DeepEqual(filterProfile, reply) { - t.Errorf("Expecting : %+v, received: %+v", filterProfile, reply) + } else if !reflect.DeepEqual(filter, reply) { + t.Errorf("Expecting : %+v, received: %+v", filter, reply) } } -func testFilterProfileRemFilterProfile(t *testing.T) { +func testFilterRemFilter(t *testing.T) { var resp string - if err := filterProfileRPC.Call("ApierV1.RemFilterProfile", &utils.TenantID{Tenant: "cgrates.org", ID: "Filter1"}, &resp); err != nil { + if err := filterRPC.Call("ApierV1.RemFilter", &utils.TenantID{Tenant: "cgrates.org", ID: "Filter1"}, &resp); err != nil { t.Error(err) } else if resp != utils.OK { t.Error("Unexpected reply returned", resp) } } -func testFilterProfileGetFilterProfileAfterRemove(t *testing.T) { - var reply *engine.FilterProfile - if err := filterProfileRPC.Call("ApierV1.GetFilterProfile", &utils.TenantID{Tenant: "cgrates.org", ID: "Filter1"}, &reply); err == nil || err.Error() != utils.ErrNotFound.Error() { +func testFilterGetFilterAfterRemove(t *testing.T) { + var reply *engine.Filter + if err := filterRPC.Call("ApierV1.GetFilter", &utils.TenantID{Tenant: "cgrates.org", ID: "Filter1"}, &reply); err == nil || err.Error() != utils.ErrNotFound.Error() { t.Error(err) } } -func testFilterProfileKillEngine(t *testing.T) { - if err := engine.KillEngine(filterProfileDelay); err != nil { +func testFilterKillEngine(t *testing.T) { + if err := engine.KillEngine(filterDelay); err != nil { t.Error(err) } } -*/ From ec5adbe93fc20cbcfcf5abf70653254e2a8c4ed1 Mon Sep 17 00:00:00 2001 From: TeoV Date: Wed, 11 Oct 2017 13:47:40 +0300 Subject: [PATCH 02/10] Updated test in loader_csv-test.go --- engine/loader_csv_test.go | 86 +++++++++++++++++++++++++++++++++++++-- engine/storage_map.go | 4 ++ engine/tp_reader.go | 2 + 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index c9c58758b..1b18440c7 100755 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -356,6 +356,9 @@ func init() { if err := csvr.LoadThresholds(); err != nil { log.Print("error in LoadThresholds:", err) } + if err := csvr.LoadFilter(); err != nil { + log.Print("error in LoadFilter:", err) + } csvr.WriteToDatabase(false, false, false) cache.Flush() dm.DataDB().LoadDataDBCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) @@ -1441,12 +1444,11 @@ func TestLoadResourceProfiles(t *testing.T) { t.Errorf("Failed to load resourceProfiles: %s", utils.ToIJSON(csvr.resProfiles)) } else if !reflect.DeepEqual(eResProfiles["cgrates.org"]["ResGroup22"], csvr.resProfiles["cgrates.org"]["ResGroup22"]) { t.Errorf("Expecting: %+v, received: %+v", eResProfiles["cgrates.org"]["ResGroup22"], csvr.resProfiles["cgrates.org"]["ResGroup22"]) - } } -func TestLoadStats(t *testing.T) { +func TestLoadStatProfiles(t *testing.T) { eStats := map[string]map[string]*utils.TPStats{ "cgrates.org": map[string]*utils.TPStats{ "Stats1": &utils.TPStats{ @@ -1479,7 +1481,7 @@ func TestLoadStats(t *testing.T) { } } -func TestLoadThresholds(t *testing.T) { +func TestLoadThresholdProfiles(t *testing.T) { eThresholds := map[string]map[string]*utils.TPThreshold{ "cgrates.org": map[string]*utils.TPThreshold{ "Threshold1": &utils.TPThreshold{ @@ -1506,3 +1508,81 @@ func TestLoadThresholds(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", eThresholds["cgrates.org"]["Threshold1"], csvr.thProfiles["cgrates.org"]["Threshold1"]) } } + +func TestLoadFilterProfiles(t *testing.T) { +} + +func TestLoadResource(t *testing.T) { + eResources := []*utils.TenantID{ + &utils.TenantID{ + Tenant: "cgrates.org", + ID: "ResGroup21", + }, + &utils.TenantID{ + Tenant: "cgrates.org", + ID: "ResGroup22", + }, + } + + if len(csvr.resources) != len(eResources) { + t.Errorf("Failed to load resources: %s", utils.ToIJSON(csvr.resources)) + } else if !reflect.DeepEqual(eResources, csvr.resources) { + t.Errorf("Expecting: %+v, received: %+v", eResources, csvr.resources) + } +} + +func TestLoadstatQueues(t *testing.T) { + eStatQueues := []*utils.TenantID{ + &utils.TenantID{ + Tenant: "cgrates.org", + ID: "Stats1", + }, + } + + if len(csvr.statQueues) != len(eStatQueues) { + t.Errorf("Failed to load statQueues: %s", utils.ToIJSON(csvr.statQueues)) + } else if !reflect.DeepEqual(eStatQueues, csvr.statQueues) { + t.Errorf("Expecting: %+v, received: %+v", eStatQueues, csvr.statQueues) + } +} + +func TestLoadThresholds(t *testing.T) { + eThresholds := []*utils.TenantID{ + &utils.TenantID{ + Tenant: "cgrates.org", + ID: "Threshold1", + }, + } + + if len(csvr.thresholds) != len(eThresholds) { + t.Errorf("Failed to load thresholds: %s", utils.ToIJSON(csvr.thresholds)) + } else if !reflect.DeepEqual(eThresholds, csvr.thresholds) { + t.Errorf("Expecting: %+v, received: %+v", eThresholds, csvr.thresholds) + } +} + +/* +func TestLoadFilters(t *testing.T) { + eFilters := []*utils.TenantID{ + &utils.TenantID{ + Tenant: "cgrates.org", + ID: "Threshold1", + }, + &utils.TenantID{ + Tenant: "cgrates.org", + ID: "Threshold1", + }, + &utils.TenantID{ + Tenant: "cgrates.org", + ID: "Threshold1", + }, + } + + if len(csvr.filters) != len(eFilters) { + t.Errorf("Failed to load thresholds: %s", utils.ToIJSON(csvr.thresholds)) + } else if !reflect.DeepEqual(eThresholds, csvr.thresholds) { + t.Errorf("Expecting: %+v, received: %+v", eFilters, csvr.thresholds) + } + +} +*/ diff --git a/engine/storage_map.go b/engine/storage_map.go index 135ea6332..5e9b5c8f8 100755 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -317,7 +317,11 @@ func (ms *MapStorage) HasData(categ, subject string) (bool, error) { switch categ { case utils.DESTINATION_PREFIX, utils.RATING_PLAN_PREFIX, utils.RATING_PROFILE_PREFIX, utils.ACTION_PREFIX, utils.ACTION_PLAN_PREFIX, utils.ACCOUNT_PREFIX, utils.DERIVEDCHARGERS_PREFIX, +<<<<<<< HEAD utils.ResourcesPrefix, utils.StatQueuePrefix, utils.ThresholdPrefix: +======= + utils.ResourcesPrefix, utils.StatQueuePrefix, utils.ThresholdProfilePrefix, utils.FilterPrefix: +>>>>>>> Add test in loader_csv_test.go for resources,statQueue,thresholds,filters _, exists := ms.dict[categ+subject] return exists, nil } diff --git a/engine/tp_reader.go b/engine/tp_reader.go index fc7feb23b..2a0651ff9 100755 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -2297,6 +2297,8 @@ func (tpr *TpReader) ShowStatistics() { log.Print("Stats: ", len(tpr.sqProfiles)) // thresholds log.Print("Thresholds: ", len(tpr.thProfiles)) + // thresholds + log.Print("Filters: ", len(tpr.flProfiles)) } // Returns the identities loaded for a specific category, useful for cache reloads From 4f4ba6ed736618e047f117f567f2f426c7aeb03e Mon Sep 17 00:00:00 2001 From: TeoV Date: Wed, 11 Oct 2017 17:01:23 +0300 Subject: [PATCH 03/10] Add Async option for Thresholds --- engine/loader_csv_test.go | 68 ++++++++++++++++++++++++++++++++------- engine/model_helpers.go | 17 +++++++--- engine/models.go | 1 + engine/storage_map.go | 6 +--- engine/thresholds.go | 1 + engine/tp_reader.go | 15 +++++++-- utils/apitpdata.go | 1 + 7 files changed, 85 insertions(+), 24 deletions(-) diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 1b18440c7..4452bd529 100755 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -18,13 +18,12 @@ along with this program. If not, see package engine import ( + "github.com/cgrates/cgrates/cache" + "github.com/cgrates/cgrates/utils" "log" "reflect" "testing" "time" - - "github.com/cgrates/cgrates/cache" - "github.com/cgrates/cgrates/utils" ) var ( @@ -1509,8 +1508,52 @@ func TestLoadThresholdProfiles(t *testing.T) { } } +/* func TestLoadFilterProfiles(t *testing.T) { + eFilters := map[string]map[string]*utils.TPFilter{ + "cgrates.org": map[string]*utils.TPFilter{ + "FLTR_1": &utils.TPFilter{ + TPid: testTPID, + Tenant: "cgrates.org", + ID: "FLTR_1", + FilterType: "*string", + FilterFieldName: "Account", + FilterFielValues: []string{"1001", "1002"}, + }, + "FLTR_ACNT_dan": &utils.TPFilter{ + TPid: testTPID, + Tenant: "cgrates.org", + ID: "FLTR_ACNT_dan", + FilterType: "*string", + FilterFieldName: "Account", + FilterFielValues: []string{"dan"}, + }, + "FLTR_DST_NL": &utils.TPFilter{ + TPid: testTPID, + Tenant: "cgrates.org", + ID: "FLTR_DST_NL", + FilterType: "*destinations", + FilterFieldName: "Destination", + FilterFielValues: []string{"DST_NL"}, + }, + "FLTR_DST_DE": &utils.TPFilter{ + TPid: testTPID, + Tenant: "cgrates.org", + ID: "FLTR_DST_DE", + FilterType: "*destinations", + FilterFieldName: "Destination", + FilterFielValues: []string{"DST_DE"}, + }, + }, + } + if len(csvr.flProfiles["cgrates.org"]) != len(eFilters["cgrates.org"]) { + t.Errorf("Failed to load FilterProfiles: %s", utils.ToIJSON(csvr.flProfiles)) + } else if !reflect.DeepEqual(eFilters["cgrates.org"]["FLTR_1"], csvr.flProfiles["cgrates.org"]["FLTR_1"]) { + t.Errorf("Expecting: %+v, received: %+v", eFilters["cgrates.org"]["FLTR_1"], csvr.flProfiles["cgrates.org"]["FLTR_1"]) + } + } +*/ func TestLoadResource(t *testing.T) { eResources := []*utils.TenantID{ @@ -1561,28 +1604,29 @@ func TestLoadThresholds(t *testing.T) { } } -/* func TestLoadFilters(t *testing.T) { eFilters := []*utils.TenantID{ &utils.TenantID{ Tenant: "cgrates.org", - ID: "Threshold1", + ID: "FLTR_1", }, &utils.TenantID{ Tenant: "cgrates.org", - ID: "Threshold1", + ID: "FLTR_ACNT_dan", }, &utils.TenantID{ Tenant: "cgrates.org", - ID: "Threshold1", + ID: "FLTR_DST_DE", + }, + &utils.TenantID{ + Tenant: "cgrates.org", + ID: "FLTR_DST_NL", }, } if len(csvr.filters) != len(eFilters) { - t.Errorf("Failed to load thresholds: %s", utils.ToIJSON(csvr.thresholds)) - } else if !reflect.DeepEqual(eThresholds, csvr.thresholds) { - t.Errorf("Expecting: %+v, received: %+v", eFilters, csvr.thresholds) + t.Errorf("Failed to load filters: %s", utils.ToIJSON(csvr.filters)) + } else if !reflect.DeepEqual(eFilters, csvr.filters) { + t.Errorf("Expecting: %+v, received: %+v", eFilters, csvr.filters) } - } -*/ diff --git a/engine/model_helpers.go b/engine/model_helpers.go index 1e344e0fd..dbab90660 100755 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -2138,6 +2138,7 @@ func (tps TpThresholdS) AsTPThreshold() (result []*utils.TPThreshold) { Blocker: tp.Blocker, Recurrent: tp.Recurrent, MinSleep: tp.MinSleep, + Async: tp.Async, } } if tp.ActionIDs != "" { @@ -2188,6 +2189,7 @@ func APItoModelTPThreshold(th *utils.TPThreshold) (mdls TpThresholdS) { mdl.Weight = th.Weight mdl.Recurrent = th.Recurrent mdl.MinSleep = th.MinSleep + mdl.Async = th.Async if th.ActivationInterval != nil { if th.ActivationInterval.ActivationTime != "" { mdl.ActivationInterval = th.ActivationInterval.ActivationTime @@ -2226,6 +2228,7 @@ func APItoThresholdProfile(tpTH *utils.TPThreshold, timezone string) (th *Thresh Recurrent: tpTH.Recurrent, Weight: tpTH.Weight, Blocker: tpTH.Blocker, + Async: tpTH.Async, Filters: make([]*Filter, len(tpTH.Filters)), } if tpTH.MinSleep != "" { @@ -2260,13 +2263,17 @@ func (tps TpFilterS) AsTPFilter() (result []*utils.TPFilter) { th, found := mst[tp.ID] if !found { th = &utils.TPFilter{ - TPid: tp.Tpid, - Tenant: tp.Tenant, - ID: tp.ID, - FilterType: tp.Type, - FilterFieldName: tp.Name, + TPid: tp.Tpid, + Tenant: tp.Tenant, + ID: tp.ID, } } + if tp.Type != "" { + th.FilterType = tp.Type + } + if tp.Name != "" { + th.FilterFieldName = tp.Name + } if tp.Values != "" { th.FilterFielValues = append(th.FilterFielValues, strings.Split(tp.Values, utils.INFIELD_SEP)...) } diff --git a/engine/models.go b/engine/models.go index 1f01820e9..82cbad039 100755 --- a/engine/models.go +++ b/engine/models.go @@ -514,6 +514,7 @@ type TpThreshold struct { Blocker bool `index:"8" re:""` Weight float64 `index:"9" re:"\d+\.?\d*"` ActionIDs string `index:"10" re:""` + Async bool `index:"8" re:""` CreatedAt time.Time } diff --git a/engine/storage_map.go b/engine/storage_map.go index 5e9b5c8f8..64aed54fd 100755 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -317,11 +317,7 @@ func (ms *MapStorage) HasData(categ, subject string) (bool, error) { switch categ { case utils.DESTINATION_PREFIX, utils.RATING_PLAN_PREFIX, utils.RATING_PROFILE_PREFIX, utils.ACTION_PREFIX, utils.ACTION_PLAN_PREFIX, utils.ACCOUNT_PREFIX, utils.DERIVEDCHARGERS_PREFIX, -<<<<<<< HEAD - utils.ResourcesPrefix, utils.StatQueuePrefix, utils.ThresholdPrefix: -======= - utils.ResourcesPrefix, utils.StatQueuePrefix, utils.ThresholdProfilePrefix, utils.FilterPrefix: ->>>>>>> Add test in loader_csv_test.go for resources,statQueue,thresholds,filters + utils.ResourcesPrefix, utils.StatQueuePrefix, utils.ThresholdPrefix, utils.FilterPrefix: _, exists := ms.dict[categ+subject] return exists, nil } diff --git a/engine/thresholds.go b/engine/thresholds.go index ac962016d..41fa7c4e5 100644 --- a/engine/thresholds.go +++ b/engine/thresholds.go @@ -42,6 +42,7 @@ type ThresholdProfile struct { Blocker bool // blocker flag to stop processing on filters matched Weight float64 // Weight to sort the thresholds ActionIDs []string + Async bool } func (tp *ThresholdProfile) TenantID() string { diff --git a/engine/tp_reader.go b/engine/tp_reader.go index 2a0651ff9..f2f6588f5 100755 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -2106,7 +2106,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err } } if verbose { - log.Print("Filters:") + log.Print("FilterProfile:") } for _, mpID := range tpr.flProfiles { for _, tpTH := range mpID { @@ -2122,6 +2122,17 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err } } } + if verbose { + log.Print("Filters:") + } + for _, thd := range tpr.filters { + if err = tpr.dataStorage.SetFilter(&Filter{Tenant: thd.Tenant, ID: thd.ID}); err != nil { + return err + } + if verbose { + log.Print("\t", thd.TenantID()) + } + } if verbose { log.Print("Timings:") } @@ -2297,7 +2308,7 @@ func (tpr *TpReader) ShowStatistics() { log.Print("Stats: ", len(tpr.sqProfiles)) // thresholds log.Print("Thresholds: ", len(tpr.thProfiles)) - // thresholds + // filters log.Print("Filters: ", len(tpr.flProfiles)) } diff --git a/utils/apitpdata.go b/utils/apitpdata.go index df349037c..91ac303dd 100755 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -1366,6 +1366,7 @@ type TPThreshold struct { Blocker bool // blocker flag to stop processing on filters matched Weight float64 // Weight to sort the thresholds ActionIDs []string + Async bool } type TPFilter struct { From 68ca42c4c0e5911a8355d941e4259b29c2a226dd Mon Sep 17 00:00:00 2001 From: TeoV Date: Thu, 12 Oct 2017 08:02:41 +0300 Subject: [PATCH 04/10] Add Async to Thresholds --- apier/v1/tpthresholds_it_test.go | 1 + .../mysql/create_tariffplan_tables.sql | 1 + .../postgres/create_tariffplan_tables.sql | 1 + data/tariffplans/testtp/Thresholds.csv | 4 +- data/tariffplans/tutorial/Thresholds.csv | 2 +- engine/filter_test.go | 16 +++---- engine/filterindexer.go | 2 +- engine/filters.go | 46 +++++++++---------- engine/libstats.go | 2 +- engine/loader_csv_test.go | 10 ++-- engine/model_helpers.go | 21 ++++----- engine/resources.go | 2 +- engine/thresholds.go | 2 +- 13 files changed, 55 insertions(+), 55 deletions(-) diff --git a/apier/v1/tpthresholds_it_test.go b/apier/v1/tpthresholds_it_test.go index 55ee3c066..c476dcbc9 100644 --- a/apier/v1/tpthresholds_it_test.go +++ b/apier/v1/tpthresholds_it_test.go @@ -147,6 +147,7 @@ func testTPThreholdSetTPThrehold(t *testing.T) { Blocker: true, Weight: 10, ActionIDs: []string{"Thresh1", "Thresh2"}, + Async: true, } var result string if err := tpThresholdRPC.Call("ApierV1.SetTPThreshold", tpThreshold, &result); err != nil { diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index f4ec7fd69..ba430c7c9 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -465,6 +465,7 @@ CREATE TABLE tp_thresholds ( `blocker` BOOLEAN NOT NULL, `weight` decimal(8,2) NOT NULL, `action_ids` varchar(64) NOT NULL, + `async` BOOLEAN NOT NULL, `created_at` TIMESTAMP, PRIMARY KEY (`pk`), KEY `tpid` (`tpid`), diff --git a/data/storage/postgres/create_tariffplan_tables.sql b/data/storage/postgres/create_tariffplan_tables.sql index b53256962..33d5fa9b4 100644 --- a/data/storage/postgres/create_tariffplan_tables.sql +++ b/data/storage/postgres/create_tariffplan_tables.sql @@ -460,6 +460,7 @@ CREATE TABLE tp_thresholds ( "blocker" BOOLEAN NOT NULL, "weight" decimal(8,2) NOT NULL, "action_ids" varchar(64) NOT NULL, + "async" BOOLEAN NOT NULL, "created_at" TIMESTAMP WITH TIME ZONE ); CREATE INDEX tp_thresholds_idx ON tp_thresholds (tpid); diff --git a/data/tariffplans/testtp/Thresholds.csv b/data/tariffplans/testtp/Thresholds.csv index a422ddaa6..9f6286555 100644 --- a/data/tariffplans/testtp/Thresholds.csv +++ b/data/tariffplans/testtp/Thresholds.csv @@ -1,2 +1,2 @@ -#Tenant[0],Id[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5],Recurrent[6],MinSleep[7],Blocker[8],Weight[9],ActionIDs[10] -cgrates.org,Threshold1,*string,Account,1001;1002,2014-07-29T15:00:00Z,true,1s,true,10,THRESH1;THRESH2 +#Tenant[0],Id[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5],Recurrent[6],MinSleep[7],Blocker[8],Weight[9],ActionIDs[10],Async[11] +cgrates.org,Threshold1,*string,Account,1001;1002,2014-07-29T15:00:00Z,true,1s,true,10,THRESH1;THRESH2,true diff --git a/data/tariffplans/tutorial/Thresholds.csv b/data/tariffplans/tutorial/Thresholds.csv index 8e55bef14..9f6286555 100644 --- a/data/tariffplans/tutorial/Thresholds.csv +++ b/data/tariffplans/tutorial/Thresholds.csv @@ -1,2 +1,2 @@ #Tenant[0],Id[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5],Recurrent[6],MinSleep[7],Blocker[8],Weight[9],ActionIDs[10],Async[11] -cgrates.org,Threshold1,*string,Account,1001;1002,2014-07-29T15:00:00Z,true,1s,true,10,THRESH1;THRESH2 +cgrates.org,Threshold1,*string,Account,1001;1002,2014-07-29T15:00:00Z,true,1s,true,10,THRESH1;THRESH2,true diff --git a/engine/filter_test.go b/engine/filter_test.go index 6a2abdcdd..2d96fa62f 100644 --- a/engine/filter_test.go +++ b/engine/filter_test.go @@ -29,13 +29,13 @@ func TestReqFilterPassString(t *testing.T) { cd := &CallDescriptor{Direction: "*out", Category: "call", Tenant: "cgrates.org", Subject: "dan", Destination: "+4986517174963", TimeStart: time.Date(2013, time.October, 7, 14, 50, 0, 0, time.UTC), TimeEnd: time.Date(2013, time.October, 7, 14, 52, 12, 0, time.UTC), DurationIndex: 132 * time.Second, ExtraFields: map[string]string{"navigation": "off"}} - rf := &Filter{Type: MetaString, FieldName: "Category", Values: []string{"call"}} + rf := &RequestFilter{Type: MetaString, FieldName: "Category", Values: []string{"call"}} if passes, err := rf.passString(cd, ""); err != nil { t.Error(err) } else if !passes { t.Error("Not passes filter") } - rf = &Filter{Type: MetaString, FieldName: "Category", Values: []string{"cal"}} + rf = &RequestFilter{Type: MetaString, FieldName: "Category", Values: []string{"cal"}} if passes, err := rf.passString(cd, ""); err != nil { t.Error(err) } else if passes { @@ -47,37 +47,37 @@ func TestReqFilterPassStringPrefix(t *testing.T) { cd := &CallDescriptor{Direction: "*out", Category: "call", Tenant: "cgrates.org", Subject: "dan", Destination: "+4986517174963", TimeStart: time.Date(2013, time.October, 7, 14, 50, 0, 0, time.UTC), TimeEnd: time.Date(2013, time.October, 7, 14, 52, 12, 0, time.UTC), DurationIndex: 132 * time.Second, ExtraFields: map[string]string{"navigation": "off"}} - rf := &Filter{Type: MetaStringPrefix, FieldName: "Category", Values: []string{"call"}} + rf := &RequestFilter{Type: MetaStringPrefix, FieldName: "Category", Values: []string{"call"}} if passes, err := rf.passStringPrefix(cd, ""); err != nil { t.Error(err) } else if !passes { t.Error("Not passes filter") } - rf = &Filter{Type: MetaStringPrefix, FieldName: "Category", Values: []string{"premium"}} + rf = &RequestFilter{Type: MetaStringPrefix, FieldName: "Category", Values: []string{"premium"}} if passes, err := rf.passStringPrefix(cd, ""); err != nil { t.Error(err) } else if passes { t.Error("Passes filter") } - rf = &Filter{Type: MetaStringPrefix, FieldName: "Destination", Values: []string{"+49"}} + rf = &RequestFilter{Type: MetaStringPrefix, FieldName: "Destination", Values: []string{"+49"}} if passes, err := rf.passStringPrefix(cd, ""); err != nil { t.Error(err) } else if !passes { t.Error("Not passes filter") } - rf = &Filter{Type: MetaStringPrefix, FieldName: "Destination", Values: []string{"+499"}} + rf = &RequestFilter{Type: MetaStringPrefix, FieldName: "Destination", Values: []string{"+499"}} if passes, err := rf.passStringPrefix(cd, ""); err != nil { t.Error(err) } else if passes { t.Error("Passes filter") } - rf = &Filter{Type: MetaStringPrefix, FieldName: "navigation", Values: []string{"off"}} + rf = &RequestFilter{Type: MetaStringPrefix, FieldName: "navigation", Values: []string{"off"}} if passes, err := rf.passStringPrefix(cd, "ExtraFields"); err != nil { t.Error(err) } else if !passes { t.Error("Not passes filter") } - rf = &Filter{Type: MetaStringPrefix, FieldName: "nonexisting", Values: []string{"off"}} + rf = &RequestFilter{Type: MetaStringPrefix, FieldName: "nonexisting", Values: []string{"off"}} if passing, err := rf.passStringPrefix(cd, "ExtraFields"); err != nil { t.Error(err) } else if passing { diff --git a/engine/filterindexer.go b/engine/filterindexer.go index da76f4e28..b456f8366 100644 --- a/engine/filterindexer.go +++ b/engine/filterindexer.go @@ -50,7 +50,7 @@ func (rfi *ReqFilterIndexer) ChangedKeys() utils.StringMap { } // IndexFilters parses reqFltrs, adding itemID in the indexes and marks the changed keys in chngdIndxKeys -func (rfi *ReqFilterIndexer) IndexFilters(itemID string, reqFltrs []*Filter) { +func (rfi *ReqFilterIndexer) IndexFilters(itemID string, reqFltrs []*RequestFilter) { var hasMetaString bool for _, fltr := range reqFltrs { if fltr.Type != MetaString { diff --git a/engine/filters.go b/engine/filters.go index 8729c3a3a..98bf26fef 100644 --- a/engine/filters.go +++ b/engine/filters.go @@ -1,17 +1,14 @@ /* Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments Copyright (C) ITsysCOM GmbH - This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - You should have received a copy of the GNU General Public License along with this program. If not, see */ @@ -40,7 +37,7 @@ const ( MetaMaxCapPrefix = "*max_" ) -func NewFilter(rfType, fieldName string, vals []string) (*Filter, error) { +func NewRequestFilter(rfType, fieldName string, vals []string) (*RequestFilter, error) { if !utils.IsSliceMember([]string{MetaStringPrefix, MetaTimings, MetaRSRFields, MetaStatS, MetaDestinations}, rfType) { return nil, fmt.Errorf("Unsupported filter Type: %s", rfType) } @@ -50,7 +47,7 @@ func NewFilter(rfType, fieldName string, vals []string) (*Filter, error) { if len(vals) == 0 && utils.IsSliceMember([]string{MetaStringPrefix, MetaTimings, MetaRSRFields, MetaDestinations, MetaDestinations}, rfType) { return nil, fmt.Errorf("Values is mandatory for Type: %s", rfType) } - rf := &Filter{Type: rfType, FieldName: fieldName, Values: vals} + rf := &RequestFilter{Type: rfType, FieldName: fieldName, Values: vals} if err := rf.CompileValues(); err != nil { return nil, err } @@ -63,24 +60,25 @@ type RFStatSThreshold struct { ThresholdValue float64 } -// Filter filters requests coming into various places +// RequestFilter filters requests coming into various places // Pass rule: default negative, one mathing rule should pass the filter -type Filter struct { - Tenant string - ID string - Type string // Filter type (*string, *timing, *rsr_filters, *cdr_stats) - FieldName string // Name of the field providing us the Values to check (used in case of some ) - Values []string // Filter definition - rsrFields utils.RSRFields // Cache here the RSRFilter Values - statSThresholds []*RFStatSThreshold // Cached compiled RFStatsThreshold out of Values +type RequestFilter struct { + Type string // Filter type (*string, *timing, *rsr_filters, *cdr_stats) + FieldName string // Name of the field providing us the Values to check (used in case of some ) + Values []string // Filter definition + ActivationInterval *utils.ActivationInterval + rsrFields utils.RSRFields // Cache here the RSRFilter Values + statSThresholds []*RFStatSThreshold // Cached compiled RFStatsThreshold out of Values } -func (flt *Filter) TenantID() string { - return utils.ConcatenatedKey(flt.Tenant, flt.ID) +type Filter struct { + Tenant string + ID string + ReqFilters []*RequestFilter } // Separate method to compile RSR fields -func (rf *Filter) CompileValues() (err error) { +func (rf *RequestFilter) CompileValues() (err error) { if rf.Type == MetaRSRFields { if rf.rsrFields, err = utils.ParseRSRFieldsFromSlice(rf.Values); err != nil { return @@ -110,7 +108,7 @@ func (rf *Filter) CompileValues() (err error) { } // Pass is the method which should be used from outside. -func (fltr *Filter) Pass(req interface{}, extraFieldsLabel string, rpcClnt rpcclient.RpcClientConnection) (bool, error) { +func (fltr *RequestFilter) Pass(req interface{}, extraFieldsLabel string, rpcClnt rpcclient.RpcClientConnection) (bool, error) { switch fltr.Type { case MetaString: return fltr.passString(req, extraFieldsLabel) @@ -129,7 +127,7 @@ func (fltr *Filter) Pass(req interface{}, extraFieldsLabel string, rpcClnt rpccl } } -func (fltr *Filter) passString(req interface{}, extraFieldsLabel string) (bool, error) { +func (fltr *RequestFilter) passString(req interface{}, extraFieldsLabel string) (bool, error) { strVal, err := utils.ReflectFieldAsString(req, fltr.FieldName, extraFieldsLabel) if err != nil { if err == utils.ErrNotFound { @@ -145,7 +143,7 @@ func (fltr *Filter) passString(req interface{}, extraFieldsLabel string) (bool, return false, nil } -func (fltr *Filter) passStringPrefix(req interface{}, extraFieldsLabel string) (bool, error) { +func (fltr *RequestFilter) passStringPrefix(req interface{}, extraFieldsLabel string) (bool, error) { strVal, err := utils.ReflectFieldAsString(req, fltr.FieldName, extraFieldsLabel) if err != nil { if err == utils.ErrNotFound { @@ -162,11 +160,11 @@ func (fltr *Filter) passStringPrefix(req interface{}, extraFieldsLabel string) ( } // ToDo when Timings will be available in DataDb -func (fltr *Filter) passTimings(req interface{}, extraFieldsLabel string) (bool, error) { +func (fltr *RequestFilter) passTimings(req interface{}, extraFieldsLabel string) (bool, error) { return false, utils.ErrNotImplemented } -func (fltr *Filter) passDestinations(req interface{}, extraFieldsLabel string) (bool, error) { +func (fltr *RequestFilter) passDestinations(req interface{}, extraFieldsLabel string) (bool, error) { dst, err := utils.ReflectFieldAsString(req, fltr.FieldName, extraFieldsLabel) if err != nil { if err == utils.ErrNotFound { @@ -188,7 +186,7 @@ func (fltr *Filter) passDestinations(req interface{}, extraFieldsLabel string) ( return false, nil } -func (fltr *Filter) passRSRFields(req interface{}, extraFieldsLabel string) (bool, error) { +func (fltr *RequestFilter) passRSRFields(req interface{}, extraFieldsLabel string) (bool, error) { for _, rsrFld := range fltr.rsrFields { if strVal, err := utils.ReflectFieldAsString(req, rsrFld.Id, extraFieldsLabel); err != nil { if err == utils.ErrNotFound { @@ -202,7 +200,7 @@ func (fltr *Filter) passRSRFields(req interface{}, extraFieldsLabel string) (boo return false, nil } -func (fltr *Filter) passStatS(req interface{}, extraFieldsLabel string, stats rpcclient.RpcClientConnection) (bool, error) { +func (fltr *RequestFilter) passStatS(req interface{}, extraFieldsLabel string, stats rpcclient.RpcClientConnection) (bool, error) { if stats == nil || reflect.ValueOf(stats).IsNil() { return false, errors.New("Missing StatS information") } diff --git a/engine/libstats.go b/engine/libstats.go index 8019bdadc..2bb680486 100755 --- a/engine/libstats.go +++ b/engine/libstats.go @@ -31,7 +31,7 @@ import ( type StatQueueProfile struct { Tenant string ID string // QueueID - Filters []*Filter + Filters []*RequestFilter ActivationInterval *utils.ActivationInterval // Activation interval QueueLength int TTL time.Duration diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 4452bd529..ef0ea92ba 100755 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -277,8 +277,8 @@ cgrates.org,Stats1,*string,Account,1001;1002,2014-07-29T15:00:00Z,100,1s,*asr;*a ` thresholds = ` -#Tenant[0],Id[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5],Recurrent[6],MinSleep[7],Blocker[8],Weight[9],ActionIDs[10] -cgrates.org,Threshold1,*string,Account,1001;1002,2014-07-29T15:00:00Z,true,1s,true,10,THRESH1;THRESH2 +#Tenant[0],Id[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5],Recurrent[6],MinSleep[7],Blocker[8],Weight[9],ActionIDs[10],Async[11] +cgrates.org,Threshold1,*string,Account,1001;1002,2014-07-29T15:00:00Z,true,1s,true,10,THRESH1;THRESH2,true ` filters = ` #Tenant[0],ID[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4] @@ -1498,6 +1498,7 @@ func TestLoadThresholdProfiles(t *testing.T) { Blocker: true, Weight: 10, ActionIDs: []string{"THRESH1", "THRESH2"}, + Async: true, }, }, } @@ -1555,6 +1556,7 @@ func TestLoadFilterProfiles(t *testing.T) { } */ +/* func TestLoadResource(t *testing.T) { eResources := []*utils.TenantID{ &utils.TenantID{ @@ -1566,13 +1568,13 @@ func TestLoadResource(t *testing.T) { ID: "ResGroup22", }, } - if len(csvr.resources) != len(eResources) { t.Errorf("Failed to load resources: %s", utils.ToIJSON(csvr.resources)) } else if !reflect.DeepEqual(eResources, csvr.resources) { t.Errorf("Expecting: %+v, received: %+v", eResources, csvr.resources) } } +*/ func TestLoadstatQueues(t *testing.T) { eStatQueues := []*utils.TenantID{ @@ -1604,6 +1606,7 @@ func TestLoadThresholds(t *testing.T) { } } +/* func TestLoadFilters(t *testing.T) { eFilters := []*utils.TenantID{ &utils.TenantID{ @@ -1630,3 +1633,4 @@ func TestLoadFilters(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", eFilters, csvr.filters) } } +*/ diff --git a/engine/model_helpers.go b/engine/model_helpers.go index dbab90660..b2e9bdef8 100755 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -1934,7 +1934,7 @@ func APItoResource(tpRL *utils.TPResource, timezone string) (rp *ResourceProfile Weight: tpRL.Weight, Blocker: tpRL.Blocker, Stored: tpRL.Stored, - Filters: make([]*Filter, len(tpRL.Filters)), + Filters: make([]*RequestFilter, len(tpRL.Filters)), } if tpRL.UsageTTL != "" { if rp.UsageTTL, err = utils.ParseDurationWithSecs(tpRL.UsageTTL); err != nil { @@ -1942,7 +1942,7 @@ func APItoResource(tpRL *utils.TPResource, timezone string) (rp *ResourceProfile } } for i, f := range tpRL.Filters { - rf := &Filter{Type: f.Type, FieldName: f.FieldName, Values: f.Values} + rf := &RequestFilter{Type: f.Type, FieldName: f.FieldName, Values: f.Values} if err := rf.CompileValues(); err != nil { return nil, err } @@ -2095,7 +2095,7 @@ func APItoStats(tpST *utils.TPStats, timezone string) (st *StatQueueProfile, err Blocker: tpST.Blocker, Stored: tpST.Stored, MinItems: tpST.MinItems, - Filters: make([]*Filter, len(tpST.Filters)), + Filters: make([]*RequestFilter, len(tpST.Filters)), } if tpST.TTL != "" { if st.TTL, err = utils.ParseDurationWithSecs(tpST.TTL); err != nil { @@ -2110,7 +2110,7 @@ func APItoStats(tpST *utils.TPStats, timezone string) (st *StatQueueProfile, err } for i, f := range tpST.Filters { - rf := &Filter{Type: f.Type, FieldName: f.FieldName, Values: f.Values} + rf := &RequestFilter{Type: f.Type, FieldName: f.FieldName, Values: f.Values} if err := rf.CompileValues(); err != nil { return nil, err } @@ -2229,7 +2229,7 @@ func APItoThresholdProfile(tpTH *utils.TPThreshold, timezone string) (th *Thresh Weight: tpTH.Weight, Blocker: tpTH.Blocker, Async: tpTH.Async, - Filters: make([]*Filter, len(tpTH.Filters)), + Filters: make([]*RequestFilter, len(tpTH.Filters)), } if tpTH.MinSleep != "" { if th.MinSleep, err = utils.ParseDurationWithSecs(tpTH.MinSleep); err != nil { @@ -2241,7 +2241,7 @@ func APItoThresholdProfile(tpTH *utils.TPThreshold, timezone string) (th *Thresh } for i, f := range tpTH.Filters { - rf := &Filter{Type: f.Type, FieldName: f.FieldName, Values: f.Values} + rf := &RequestFilter{Type: f.Type, FieldName: f.FieldName, Values: f.Values} if err := rf.CompileValues(); err != nil { return nil, err } @@ -2308,13 +2308,8 @@ func APItoModelTPFilter(th *utils.TPFilter) (mdls TpFilterS) { func APItoFilter(tpTH *utils.TPFilter) (th *Filter, err error) { th = &Filter{ - Tenant: tpTH.Tenant, - ID: tpTH.ID, - FieldName: tpTH.FilterFieldName, - Type: tpTH.FilterType, - } - for _, ati := range tpTH.FilterFielValues { - th.Values = append(th.Values, ati) + Tenant: tpTH.Tenant, + ID: tpTH.ID, } return th, nil } diff --git a/engine/resources.go b/engine/resources.go index 090617bdd..e0b577bc4 100755 --- a/engine/resources.go +++ b/engine/resources.go @@ -41,7 +41,7 @@ func init() { type ResourceProfile struct { Tenant string ID string // identifier of this resource - Filters []*Filter // filters for the request + Filters []*RequestFilter // filters for the request ActivationInterval *utils.ActivationInterval // time when this resource becomes active and expires UsageTTL time.Duration // auto-expire the usage after this duration Limit float64 // limit value diff --git a/engine/thresholds.go b/engine/thresholds.go index 41fa7c4e5..42654bd45 100644 --- a/engine/thresholds.go +++ b/engine/thresholds.go @@ -35,7 +35,7 @@ import ( type ThresholdProfile struct { Tenant string ID string - Filters []*Filter // Filters for the request + Filters []*RequestFilter // Filters for the request ActivationInterval *utils.ActivationInterval // Time when this limit becomes active and expires Recurrent bool MinSleep time.Duration From 12262fb11faf00562fced0701975fa196713baca Mon Sep 17 00:00:00 2001 From: TeoV Date: Thu, 12 Oct 2017 11:09:02 +0300 Subject: [PATCH 05/10] Refactoring filters --- apier/v1/filters_it_test.go | 31 ++++++++---- apier/v1/resourcesv1_it_test.go | 10 ++-- apier/v1/stats_it_test.go | 12 ++--- data/tariffplans/tutorial/Filters.csv | 2 +- engine/filter_test.go | 13 ++--- engine/filters.go | 10 ++-- engine/model_helpers.go | 58 ++++++++++++--------- engine/model_helpers_test.go | 72 ++++++++++++++++----------- engine/models.go | 16 +++--- engine/onstor_it_test.go | 30 ++++++----- engine/resources_test.go | 18 +++---- utils/apitpdata.go | 11 ++-- 12 files changed, 158 insertions(+), 125 deletions(-) diff --git a/apier/v1/filters_it_test.go b/apier/v1/filters_it_test.go index 8f015e2db..e690e30b9 100644 --- a/apier/v1/filters_it_test.go +++ b/apier/v1/filters_it_test.go @@ -127,11 +127,15 @@ func testFilterGetFilterBeforeSet(t *testing.T) { func testFilterSetFilter(t *testing.T) { filter = &engine.Filter{ - Tenant: "cgrates.org", - ID: "Filter1", - Type: "*string_prefix", - FieldName: "Account", - Values: []string{"10", "20"}, + Tenant: "cgrates.org", + ID: "Filter1", + Filters: []*engine.RequestFilter{ + &engine.RequestFilter{ + FieldName: "*string", + Type: "Account", + Values: []string{"1001", "1002"}, + }, + }, } var result string @@ -152,12 +156,17 @@ func testFilterGetFilterAfterSet(t *testing.T) { } func testFilterUpdateFilter(t *testing.T) { - filter = &engine.Filter{ - Tenant: "cgrates.org", - ID: "Filter1", - Type: "*string_prefix", - FieldName: "Destination", - Values: []string{"1001", "1002"}, + filter.Filters = []*engine.RequestFilter{ + &engine.RequestFilter{ + FieldName: "*string", + Type: "Account", + Values: []string{"1001", "1002"}, + }, + &engine.RequestFilter{ + FieldName: "*string_prefix", + Type: "Destination", + Values: []string{"10", "20"}, + }, } var result string if err := filterRPC.Call("ApierV1.SetFilter", filter, &result); err != nil { diff --git a/apier/v1/resourcesv1_it_test.go b/apier/v1/resourcesv1_it_test.go index 24c9e213c..73bc18a63 100644 --- a/apier/v1/resourcesv1_it_test.go +++ b/apier/v1/resourcesv1_it_test.go @@ -507,8 +507,8 @@ func testV1RsSetResourceProfile(t *testing.T) { rlsConfig = &engine.ResourceProfile{ Tenant: "cgrates.org", ID: "RCFG1", - Filters: []*engine.Filter{ - &engine.Filter{ + Filters: []*engine.RequestFilter{ + &engine.RequestFilter{ Type: "type", FieldName: "Name", Values: []string{"FilterValue1", "FilterValue2"}, @@ -546,13 +546,13 @@ func testV1RsGetResourceProfileAfterSet(t *testing.T) { func testV1RsUpdateResourceProfile(t *testing.T) { var result string - rlsConfig.Filters = []*engine.Filter{ - &engine.Filter{ + rlsConfig.Filters = []*engine.RequestFilter{ + &engine.RequestFilter{ Type: "type", FieldName: "Name", Values: []string{"FilterValue1", "FilterValue2"}, }, - &engine.Filter{ + &engine.RequestFilter{ Type: "*string", FieldName: "Accout", Values: []string{"1001", "1002"}, diff --git a/apier/v1/stats_it_test.go b/apier/v1/stats_it_test.go index 9346f31e4..339c9a60f 100644 --- a/apier/v1/stats_it_test.go +++ b/apier/v1/stats_it_test.go @@ -277,8 +277,8 @@ func testV1STSSetStatQueueProfile(t *testing.T) { statConfig = &engine.StatQueueProfile{ Tenant: "cgrates.org", ID: "TEST_PROFILE1", - Filters: []*engine.Filter{ - &engine.Filter{ + Filters: []*engine.RequestFilter{ + &engine.RequestFilter{ Type: "type", FieldName: "Name", Values: []string{"FilterValue1", "FilterValue2"}, @@ -313,18 +313,18 @@ func testV1STSSetStatQueueProfile(t *testing.T) { func testV1STSUpdateStatQueueProfile(t *testing.T) { var result string - statConfig.Filters = []*engine.Filter{ - &engine.Filter{ + statConfig.Filters = []*engine.RequestFilter{ + &engine.RequestFilter{ Type: "type", FieldName: "Name", Values: []string{"FilterValue1", "FilterValue2"}, }, - &engine.Filter{ + &engine.RequestFilter{ Type: "*string", FieldName: "Accout", Values: []string{"1001", "1002"}, }, - &engine.Filter{ + &engine.RequestFilter{ Type: "*string_prefix", FieldName: "Destination", Values: []string{"10", "20"}, diff --git a/data/tariffplans/tutorial/Filters.csv b/data/tariffplans/tutorial/Filters.csv index 1edb26b5c..5a90d9838 100644 --- a/data/tariffplans/tutorial/Filters.csv +++ b/data/tariffplans/tutorial/Filters.csv @@ -1,4 +1,4 @@ -#Tenant[0],ID[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5] +#Tenant[0],ID[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4] cgrates.org,FLTR_1,*string,Account,1001;1002 cgrates.org,FLTR_1,*string_prefix,Destination,10;20 cgrates.org,FLTR_1,*rsr_fields,,Subject(~^1.*1$);Destination(1002) diff --git a/engine/filter_test.go b/engine/filter_test.go index 2d96fa62f..1c8fcd969 100644 --- a/engine/filter_test.go +++ b/engine/filter_test.go @@ -1,17 +1,14 @@ /* Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments Copyright (C) ITsysCOM GmbH - This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - You should have received a copy of the GNU General Public License along with this program. If not, see */ @@ -89,7 +86,7 @@ func TestReqFilterPassRSRFields(t *testing.T) { cd := &CallDescriptor{Direction: "*out", Category: "call", Tenant: "cgrates.org", Subject: "dan", Destination: "+4986517174963", TimeStart: time.Date(2013, time.October, 7, 14, 50, 0, 0, time.UTC), TimeEnd: time.Date(2013, time.October, 7, 14, 52, 12, 0, time.UTC), DurationIndex: 132 * time.Second, ExtraFields: map[string]string{"navigation": "off"}} - rf, err := NewFilter(MetaRSRFields, "", []string{"Tenant(~^cgr.*\\.org$)"}) + rf, err := NewRequestFilter(MetaRSRFields, "", []string{"Tenant(~^cgr.*\\.org$)"}) if err != nil { t.Error(err) } @@ -98,7 +95,7 @@ func TestReqFilterPassRSRFields(t *testing.T) { } else if !passes { t.Error("Not passing") } - rf, err = NewFilter(MetaRSRFields, "", []string{"navigation(on)"}) + rf, err = NewRequestFilter(MetaRSRFields, "", []string{"navigation(on)"}) if err != nil { t.Error(err) } @@ -107,7 +104,7 @@ func TestReqFilterPassRSRFields(t *testing.T) { } else if passes { t.Error("Passing") } - rf, err = NewFilter(MetaRSRFields, "", []string{"navigation(off)"}) + rf, err = NewRequestFilter(MetaRSRFields, "", []string{"navigation(off)"}) if err != nil { t.Error(err) } @@ -123,7 +120,7 @@ func TestReqFilterPassDestinations(t *testing.T) { cd := &CallDescriptor{Direction: "*out", Category: "call", Tenant: "cgrates.org", Subject: "dan", Destination: "+4986517174963", TimeStart: time.Date(2013, time.October, 7, 14, 50, 0, 0, time.UTC), TimeEnd: time.Date(2013, time.October, 7, 14, 52, 12, 0, time.UTC), DurationIndex: 132 * time.Second, ExtraFields: map[string]string{"navigation": "off"}} - rf, err := NewFilter(MetaDestinations, "Destination", []string{"DE"}) + rf, err := NewRequestFilter(MetaDestinations, "Destination", []string{"DE"}) if err != nil { t.Error(err) } @@ -132,7 +129,7 @@ func TestReqFilterPassDestinations(t *testing.T) { } else if !passes { t.Error("Not passing") } - rf, err = NewFilter(MetaDestinations, "Destination", []string{"RO"}) + rf, err = NewRequestFilter(MetaDestinations, "Destination", []string{"RO"}) if err != nil { t.Error(err) } diff --git a/engine/filters.go b/engine/filters.go index 98bf26fef..0a7d47abf 100644 --- a/engine/filters.go +++ b/engine/filters.go @@ -72,9 +72,13 @@ type RequestFilter struct { } type Filter struct { - Tenant string - ID string - ReqFilters []*RequestFilter + Tenant string + ID string + Filters []*RequestFilter +} + +func (flt *Filter) TenantID() string { + return utils.ConcatenatedKey(flt.Tenant, flt.ID) } // Separate method to compile RSR fields diff --git a/engine/model_helpers.go b/engine/model_helpers.go index b2e9bdef8..8ff38cb4e 100755 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -2211,10 +2211,9 @@ func APItoModelTPThreshold(th *utils.TPThreshold) (mdls TpThresholdS) { mdl.FilterFieldName = fltr.FieldName for i, val := range fltr.Values { if i != 0 { - mdl.FilterFieldValues = mdl.FilterFieldValues + utils.INFIELD_SEP + val - } else { - mdl.FilterFieldValues = val + mdl.FilterFieldValues += utils.INFIELD_SEP } + mdl.FilterFieldValues += val } mdls = append(mdls, mdl) } @@ -2268,14 +2267,11 @@ func (tps TpFilterS) AsTPFilter() (result []*utils.TPFilter) { ID: tp.ID, } } - if tp.Type != "" { - th.FilterType = tp.Type - } - if tp.Name != "" { - th.FilterFieldName = tp.Name - } - if tp.Values != "" { - th.FilterFielValues = append(th.FilterFielValues, strings.Split(tp.Values, utils.INFIELD_SEP)...) + if tp.FilterType != "" { + th.Filters = append(th.Filters, &utils.TPRequestFilter{ + Type: tp.FilterType, + FieldName: tp.FilterFieldName, + Values: strings.Split(tp.FilterFieldValues, utils.INFIELD_SEP)}) } mst[tp.ID] = th } @@ -2289,27 +2285,41 @@ func (tps TpFilterS) AsTPFilter() (result []*utils.TPFilter) { } func APItoModelTPFilter(th *utils.TPFilter) (mdls TpFilterS) { - mdl := &TpFilter{ - Tpid: th.TPid, - Tenant: th.Tenant, - ID: th.ID, - Name: th.FilterFieldName, - Type: th.FilterType, + if len(th.Filters) == 0 { + return } - for i, val := range th.FilterFielValues { - if i != 0 { - mdl.Values += utils.INFIELD_SEP + for _, fltr := range th.Filters { + mdl := &TpFilter{ + Tpid: th.TPid, + Tenant: th.Tenant, + ID: th.ID, } - mdl.Values += val + mdl.FilterType = fltr.Type + mdl.FilterFieldName = fltr.FieldName + for i, val := range fltr.Values { + if i != 0 { + mdl.FilterFieldValues += utils.INFIELD_SEP + } + mdl.FilterFieldValues += val + } + mdls = append(mdls, mdl) } - mdls = append(mdls, mdl) return + } func APItoFilter(tpTH *utils.TPFilter) (th *Filter, err error) { th = &Filter{ - Tenant: tpTH.Tenant, - ID: tpTH.ID, + Tenant: tpTH.Tenant, + ID: tpTH.ID, + Filters: make([]*RequestFilter, len(tpTH.Filters)), + } + for i, f := range tpTH.Filters { + rf := &RequestFilter{Type: f.Type, FieldName: f.FieldName, Values: f.Values} + if err := rf.CompileValues(); err != nil { + return nil, err + } + th.Filters[i] = rf } return th, nil } diff --git a/engine/model_helpers_test.go b/engine/model_helpers_test.go index 9747a7bcf..77e088b26 100755 --- a/engine/model_helpers_test.go +++ b/engine/model_helpers_test.go @@ -818,18 +818,18 @@ func TestAPItoResource(t *testing.T) { Stored: tpRL.Stored, Blocker: tpRL.Blocker, Weight: tpRL.Weight, - Filters: make([]*Filter, len(tpRL.Filters))} - eRL.Filters[0] = &Filter{Type: MetaString, + Filters: make([]*RequestFilter, len(tpRL.Filters))} + eRL.Filters[0] = &RequestFilter{Type: MetaString, FieldName: "Account", Values: []string{"1001", "1002"}} - eRL.Filters[1] = &Filter{Type: MetaStringPrefix, + eRL.Filters[1] = &RequestFilter{Type: MetaStringPrefix, FieldName: "Destination", Values: []string{"10", "20"}} - eRL.Filters[2] = &Filter{Type: MetaStatS, + eRL.Filters[2] = &RequestFilter{Type: MetaStatS, Values: []string{"CDRST1:*min_asr:34", "CDRST_1001:*min_asr:20"}, statSThresholds: []*RFStatSThreshold{ &RFStatSThreshold{QueueID: "CDRST1", ThresholdType: "*min_asr", ThresholdValue: 34}, &RFStatSThreshold{QueueID: "CDRST_1001", ThresholdType: "*min_asr", ThresholdValue: 20}, }} - eRL.Filters[3] = &Filter{Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"}, + eRL.Filters[3] = &RequestFilter{Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"}, rsrFields: utils.ParseRSRFieldsMustCompile("Subject(~^1.*1$);Destination(1002)", utils.INFIELD_SEP), } at, _ := utils.ParseTimeDetectLayout("2014-07-29T15:00:00Z", "UTC") @@ -913,7 +913,7 @@ func TestAPItoTPStats(t *testing.T) { QueueLength: tps.QueueLength, Metrics: []string{"*asr", "*acd", "*acc"}, Thresholds: []string{"THRESH1", "THRESH2"}, - Filters: make([]*Filter, len(tps.Filters)), + Filters: make([]*RequestFilter, len(tps.Filters)), Stored: tps.Stored, Blocker: tps.Blocker, Weight: 20.0, @@ -923,7 +923,7 @@ func TestAPItoTPStats(t *testing.T) { t.Errorf("Got error: %+v", err) } - eTPs.Filters[0] = &Filter{Type: MetaString, + eTPs.Filters[0] = &RequestFilter{Type: MetaString, FieldName: "Account", Values: []string{"1001", "1002"}} at, _ := utils.ParseTimeDetectLayout("2014-07-29T15:00:00Z", "UTC") eTPs.ActivationInterval = &utils.ActivationInterval{ActivationTime: at} @@ -995,7 +995,7 @@ func TestAPItoTPThreshold(t *testing.T) { eTPs := &ThresholdProfile{ ID: tps.ID, - Filters: make([]*Filter, len(tps.Filters)), + Filters: make([]*RequestFilter, len(tps.Filters)), Recurrent: tps.Recurrent, Blocker: tps.Blocker, Weight: tps.Weight, @@ -1004,7 +1004,7 @@ func TestAPItoTPThreshold(t *testing.T) { if eTPs.MinSleep, err = utils.ParseDurationWithSecs(tps.MinSleep); err != nil { t.Errorf("Got error: %+v", err) } - eTPs.Filters[0] = &Filter{Type: MetaString, + eTPs.Filters[0] = &RequestFilter{Type: MetaString, FieldName: "Account", Values: []string{"1001", "1002"}} at, _ := utils.ParseTimeDetectLayout("2014-07-29T15:00:00Z", "UTC") eTPs.ActivationInterval = &utils.ActivationInterval{ActivationTime: at} @@ -1018,20 +1018,24 @@ func TestAPItoTPThreshold(t *testing.T) { func TestTPFilterAsTPFilter(t *testing.T) { tps := []*TpFilter{ &TpFilter{ - Tpid: "TEST_TPID", - ID: "Filter1", - Type: MetaStringPrefix, - Name: "Account", - Values: "1001;1002", + Tpid: "TEST_TPID", + ID: "Filter1", + FilterType: MetaStringPrefix, + FilterFieldName: "Account", + FilterFieldValues: "1001;1002", }, } eTPs := []*utils.TPFilter{ &utils.TPFilter{ - TPid: tps[0].Tpid, - ID: tps[0].ID, - FilterType: tps[0].Type, - FilterFieldName: tps[0].Name, - FilterFielValues: []string{"1001", "1002"}, + TPid: tps[0].Tpid, + ID: tps[0].ID, + Filters: []*utils.TPRequestFilter{ + &utils.TPRequestFilter{ + Type: MetaStringPrefix, + FieldName: "Account", + Values: []string{"1001", "1002"}, + }, + }, }, } @@ -1043,20 +1047,28 @@ func TestTPFilterAsTPFilter(t *testing.T) { func TestAPItoTPFilter(t *testing.T) { tps := &utils.TPFilter{ - TPid: testTPID, - Tenant: "cgrates.org", - ID: "Filter1", - FilterType: "*string", - FilterFieldName: "Acount", - FilterFielValues: []string{"1001", "1002"}, + TPid: testTPID, + Tenant: "cgrates.org", + ID: "Filter1", + Filters: []*utils.TPRequestFilter{ + &utils.TPRequestFilter{ + FieldName: "Account", + Type: "*string", + Values: []string{"1001", "1002"}, + }, + }, } eTPs := &Filter{ - Tenant: "cgrates.org", - ID: tps.ID, - FieldName: tps.FilterFieldName, - Type: tps.FilterType, - Values: tps.FilterFielValues, + Tenant: "cgrates.org", + ID: tps.ID, + Filters: []*RequestFilter{ + &RequestFilter{ + FieldName: "Account", + Type: "*string", + Values: []string{"1001", "1002"}, + }, + }, } if st, err := APItoFilter(tps); err != nil { t.Error(err) diff --git a/engine/models.go b/engine/models.go index 82cbad039..ac3f221d8 100755 --- a/engine/models.go +++ b/engine/models.go @@ -519,12 +519,12 @@ type TpThreshold struct { } type TpFilter struct { - PK uint `gorm:"primary_key"` - Tpid string - Tenant string `index:"0" re:""` - ID string `index:"1" re:""` - Type string `index:"2" re:"^\*[A-Za-z].*"` - Name string `index:"3" re:""` - Values string `index:"4" re:""` - CreatedAt time.Time + PK uint `gorm:"primary_key"` + Tpid string + Tenant string `index:"0" re:""` + ID string `index:"1" re:""` + FilterType string `index:"2" re:"^\*[A-Za-z].*"` + FilterFieldName string `index:"3" re:""` + FilterFieldValues string `index:"4" re:""` + CreatedAt time.Time } diff --git a/engine/onstor_it_test.go b/engine/onstor_it_test.go index 4da2d6a4c..5426b58cf 100644 --- a/engine/onstor_it_test.go +++ b/engine/onstor_it_test.go @@ -797,9 +797,9 @@ func testOnStorITCacheResourceProfile(t *testing.T) { Tenant: "cgrates.org", ID: "RL_TEST", Weight: 10, - Filters: []*Filter{ - &Filter{Type: MetaString, FieldName: "Account", Values: []string{"dan", "1002"}}, - &Filter{Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"}, + Filters: []*RequestFilter{ + &RequestFilter{Type: MetaString, FieldName: "Account", Values: []string{"dan", "1002"}}, + &RequestFilter{Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"}, rsrFields: utils.ParseRSRFieldsMustCompile("Subject(~^1.*1$);Destination(1002)", utils.INFIELD_SEP), }}, ActivationInterval: &utils.ActivationInterval{ @@ -1800,9 +1800,9 @@ func testOnStorITCRUDResourceProfile(t *testing.T) { rL := &ResourceProfile{ ID: "RL_TEST2", Weight: 10, - Filters: []*Filter{ - &Filter{Type: MetaString, FieldName: "Account", Values: []string{"dan", "1002"}}, - &Filter{Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"}, + Filters: []*RequestFilter{ + &RequestFilter{Type: MetaString, FieldName: "Account", Values: []string{"dan", "1002"}}, + &RequestFilter{Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"}, rsrFields: utils.ParseRSRFieldsMustCompile("Subject(~^1.*1$);Destination(1002)", utils.INFIELD_SEP), }}, ActivationInterval: &utils.ActivationInterval{ @@ -1967,7 +1967,7 @@ func testOnStorITCRUDStatQueueProfile(t *testing.T) { sq := &StatQueueProfile{ ID: "test", ActivationInterval: &utils.ActivationInterval{}, - Filters: []*Filter{}, + Filters: []*RequestFilter{}, QueueLength: 2, TTL: timeTTL, Metrics: []string{}, @@ -2055,7 +2055,7 @@ func testOnStorITCRUDThresholdProfile(t *testing.T) { Tenant: "cgrates.org", ID: "test", ActivationInterval: &utils.ActivationInterval{}, - Filters: []*Filter{}, + Filters: []*RequestFilter{}, Recurrent: true, MinSleep: timeMinSleep, Blocker: true, @@ -2122,11 +2122,15 @@ func testOnStorITCRUDThreshold(t *testing.T) { func testOnStorITCRUDFilter(t *testing.T) { fp := &Filter{ - Tenant: "cgrates.org", - ID: "Filter1", - FieldName: "*string", - Type: "Account", - Values: []string{"1001", "1002"}, + Tenant: "cgrates.org", + ID: "Filter1", + Filters: []*RequestFilter{ + &RequestFilter{ + FieldName: "Account", + Type: "*string", + Values: []string{"1001", "1002"}, + }, + }, } if _, rcvErr := onStor.GetFilter("cgrates.org", "Filter1", true, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound { t.Error(rcvErr) diff --git a/engine/resources_test.go b/engine/resources_test.go index 3cafde6b2..ce0a5aa14 100644 --- a/engine/resources_test.go +++ b/engine/resources_test.go @@ -53,13 +53,13 @@ func TestRSRecordUsage1(t *testing.T) { rPrf: &ResourceProfile{ Tenant: "cgrates.org", ID: "RL1", - Filters: []*Filter{ - &Filter{ + Filters: []*RequestFilter{ + &RequestFilter{ Type: MetaString, FieldName: "Account", Values: []string{"1001", "1002"}, }, - &Filter{ + &RequestFilter{ Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"}, rsrFields: utils.ParseRSRFieldsMustCompile("Subject(~^1.*1$);Destination(1002)", utils.INFIELD_SEP), @@ -134,13 +134,13 @@ func TestRSRsort(t *testing.T) { ID: "RL2", rPrf: &ResourceProfile{ ID: "RL2", - Filters: []*Filter{ - &Filter{ + Filters: []*RequestFilter{ + &RequestFilter{ Type: MetaString, FieldName: "Account", Values: []string{"1001", "1002"}, }, - &Filter{ + &RequestFilter{ Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"}, rsrFields: utils.ParseRSRFieldsMustCompile("Subject(~^1.*1$);Destination(1002)", utils.INFIELD_SEP), @@ -249,13 +249,13 @@ func TestRSCacheSetGet(t *testing.T) { rPrf: &ResourceProfile{ Tenant: "cgrates.org", ID: "RL", - Filters: []*Filter{ - &Filter{ + Filters: []*RequestFilter{ + &RequestFilter{ Type: MetaString, FieldName: "Account", Values: []string{"1001", "1002"}, }, - &Filter{ + &RequestFilter{ Type: MetaRSRFields, Values: []string{"Subject(~^1.*1$)", "Destination(1002)"}, rsrFields: utils.ParseRSRFieldsMustCompile("Subject(~^1.*1$);Destination(1002)", utils.INFIELD_SEP), diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 91ac303dd..4fce18722 100755 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -1370,11 +1370,8 @@ type TPThreshold struct { } type TPFilter struct { - TPid string - Tenant string - ID string - FilterType string // Filter type (*string, *timing, *rsr_filters, *cdr_stats) - FilterFieldName string // Name of the field providing us the Values to check (used in case of some ) - FilterFielValues []string // Filter definition - + TPid string + Tenant string + ID string + Filters []*TPRequestFilter } From e83f26fe02abbae5f15191fbb60616007545aabe Mon Sep 17 00:00:00 2001 From: TeoV Date: Thu, 12 Oct 2017 11:52:48 +0300 Subject: [PATCH 06/10] Updated Loader_csv_test.go --- engine/loader_csv_test.go | 101 ++++++++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 37 deletions(-) diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index ef0ea92ba..8e89602ee 100755 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -1509,41 +1509,66 @@ func TestLoadThresholdProfiles(t *testing.T) { } } -/* func TestLoadFilterProfiles(t *testing.T) { eFilters := map[string]map[string]*utils.TPFilter{ "cgrates.org": map[string]*utils.TPFilter{ "FLTR_1": &utils.TPFilter{ - TPid: testTPID, - Tenant: "cgrates.org", - ID: "FLTR_1", - FilterType: "*string", - FilterFieldName: "Account", - FilterFielValues: []string{"1001", "1002"}, + TPid: testTPID, + Tenant: "cgrates.org", + ID: "FLTR_1", + Filters: []*utils.TPRequestFilter{ + &utils.TPRequestFilter{ + FieldName: "Account", + Type: "*string", + Values: []string{"1001", "1002"}, + }, + &utils.TPRequestFilter{ + FieldName: "Destination", + Type: "*string_prefix", + Values: []string{"10", "20"}, + }, + &utils.TPRequestFilter{ + FieldName: "", + Type: "*rsr_fields", + Values: []string{"Subject(~^1.*1$)", "Destination(1002)"}, + }, + }, }, "FLTR_ACNT_dan": &utils.TPFilter{ - TPid: testTPID, - Tenant: "cgrates.org", - ID: "FLTR_ACNT_dan", - FilterType: "*string", - FilterFieldName: "Account", - FilterFielValues: []string{"dan"}, - }, - "FLTR_DST_NL": &utils.TPFilter{ - TPid: testTPID, - Tenant: "cgrates.org", - ID: "FLTR_DST_NL", - FilterType: "*destinations", - FilterFieldName: "Destination", - FilterFielValues: []string{"DST_NL"}, + TPid: testTPID, + Tenant: "cgrates.org", + ID: "FLTR_ACNT_dan", + Filters: []*utils.TPRequestFilter{ + &utils.TPRequestFilter{ + FieldName: "Account", + Type: "*string", + Values: []string{"dan"}, + }, + }, }, "FLTR_DST_DE": &utils.TPFilter{ - TPid: testTPID, - Tenant: "cgrates.org", - ID: "FLTR_DST_DE", - FilterType: "*destinations", - FilterFieldName: "Destination", - FilterFielValues: []string{"DST_DE"}, + TPid: testTPID, + Tenant: "cgrates.org", + ID: "FLTR_DST_DE", + Filters: []*utils.TPRequestFilter{ + &utils.TPRequestFilter{ + FieldName: "Destination", + Type: "*destinations", + Values: []string{"DST_DE"}, + }, + }, + }, + "FLTR_DST_NL": &utils.TPFilter{ + TPid: testTPID, + Tenant: "cgrates.org", + ID: "FLTR_DST_NL", + Filters: []*utils.TPRequestFilter{ + &utils.TPRequestFilter{ + FieldName: "Destination", + Type: "*destinations", + Values: []string{"DST_NL"}, + }, + }, }, }, } @@ -1554,9 +1579,7 @@ func TestLoadFilterProfiles(t *testing.T) { } } -*/ -/* func TestLoadResource(t *testing.T) { eResources := []*utils.TenantID{ &utils.TenantID{ @@ -1570,11 +1593,12 @@ func TestLoadResource(t *testing.T) { } if len(csvr.resources) != len(eResources) { t.Errorf("Failed to load resources: %s", utils.ToIJSON(csvr.resources)) - } else if !reflect.DeepEqual(eResources, csvr.resources) { - t.Errorf("Expecting: %+v, received: %+v", eResources, csvr.resources) + } else if !reflect.DeepEqual(eResources[0], csvr.resources[0]) { + t.Errorf("Expecting: %+v, received: %+v", eResources[0], csvr.resources[0]) + } else if !reflect.DeepEqual(eResources[1], csvr.resources[1]) { + t.Errorf("Expecting: %+v, received: %+v", eResources[1], csvr.resources[1]) } } -*/ func TestLoadstatQueues(t *testing.T) { eStatQueues := []*utils.TenantID{ @@ -1606,7 +1630,6 @@ func TestLoadThresholds(t *testing.T) { } } -/* func TestLoadFilters(t *testing.T) { eFilters := []*utils.TenantID{ &utils.TenantID{ @@ -1626,11 +1649,15 @@ func TestLoadFilters(t *testing.T) { ID: "FLTR_DST_NL", }, } - if len(csvr.filters) != len(eFilters) { t.Errorf("Failed to load filters: %s", utils.ToIJSON(csvr.filters)) - } else if !reflect.DeepEqual(eFilters, csvr.filters) { - t.Errorf("Expecting: %+v, received: %+v", eFilters, csvr.filters) + } else if !reflect.DeepEqual(eFilters[0], csvr.filters[0]) { + t.Errorf("Expecting: %+v, received: %+v", eFilters[0], csvr.filters[0]) + } else if !reflect.DeepEqual(eFilters[1], csvr.filters[1]) { + t.Errorf("Expecting: %+v, received: %+v", eFilters[1], csvr.filters[1]) + } else if !reflect.DeepEqual(eFilters[2], csvr.filters[2]) { + t.Errorf("Expecting: %+v, received: %+v", eFilters[2], csvr.filters[2]) + } else if !reflect.DeepEqual(eFilters[3], csvr.filters[3]) { + t.Errorf("Expecting: %+v, received: %+v", eFilters[3], csvr.filters[3]) } } -*/ From 073a3530495d49bfe04dab11cc56c7d43c82d323 Mon Sep 17 00:00:00 2001 From: TeoV Date: Fri, 13 Oct 2017 10:30:06 +0300 Subject: [PATCH 07/10] Add ActivationInterval to filter and updated test --- apier/v1/filters_it_test.go | 9 ++++- apier/v1/tpfilters_it_test.go | 39 ++++++++++++------- .../mysql/create_tariffplan_tables.sql | 1 + .../postgres/create_tariffplan_tables.sql | 1 + data/tariffplans/testtp/Filters.csv | 14 +++---- data/tariffplans/tutorial/Filters.csv | 14 +++---- engine/filters.go | 7 ++-- engine/loader_csv_test.go | 38 +++++++++--------- engine/model_helpers.go | 33 +++++++++++++--- engine/model_helpers_test.go | 4 +- engine/models.go | 17 ++++---- engine/onstor_it_test.go | 6 ++- engine/tp_reader.go | 27 ++++++++++++- utils/apitpdata.go | 9 +++-- 14 files changed, 146 insertions(+), 73 deletions(-) diff --git a/apier/v1/filters_it_test.go b/apier/v1/filters_it_test.go index e690e30b9..cd3caa002 100644 --- a/apier/v1/filters_it_test.go +++ b/apier/v1/filters_it_test.go @@ -29,6 +29,7 @@ import ( "path" "reflect" "testing" + "time" ) var ( @@ -129,13 +130,17 @@ func testFilterSetFilter(t *testing.T) { filter = &engine.Filter{ Tenant: "cgrates.org", ID: "Filter1", - Filters: []*engine.RequestFilter{ + RequestFilters: []*engine.RequestFilter{ &engine.RequestFilter{ FieldName: "*string", Type: "Account", Values: []string{"1001", "1002"}, }, }, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC).Local(), + ExpiryTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC).Local(), + }, } var result string @@ -156,7 +161,7 @@ func testFilterGetFilterAfterSet(t *testing.T) { } func testFilterUpdateFilter(t *testing.T) { - filter.Filters = []*engine.RequestFilter{ + filter.RequestFilters = []*engine.RequestFilter{ &engine.RequestFilter{ FieldName: "*string", Type: "Account", diff --git a/apier/v1/tpfilters_it_test.go b/apier/v1/tpfilters_it_test.go index b302c2a03..ce1478909 100644 --- a/apier/v1/tpfilters_it_test.go +++ b/apier/v1/tpfilters_it_test.go @@ -128,12 +128,20 @@ func ttestTPFilterGetTPFilterBeforeSet(t *testing.T) { func testTPFilterSetTPFilter(t *testing.T) { tpFilter = &utils.TPFilter{ - TPid: "TP1", - Tenant: "cgrates.org", - ID: "Filter", - FilterType: "*string", - FilterFieldName: "Account", - FilterFielValues: []string{"1001", "1002"}, + TPid: "TP1", + Tenant: "cgrates.org", + ID: "Filter", + Filters: []*utils.TPRequestFilter{ + &utils.TPRequestFilter{ + Type: "*string", + FieldName: "Account", + Values: []string{"1001", "1002"}, + }, + }, + ActivationInterval: &utils.TPActivationInterval{ + ActivationTime: "2014-07-29T15:00:00Z", + ExpiryTime: "", + }, } var result string @@ -166,15 +174,18 @@ func testTPFilterGetFilterIds(t *testing.T) { } func testTPFilterUpdateTPFilter(t *testing.T) { - tpFilter = &utils.TPFilter{ - TPid: "TP1", - Tenant: "cgrates.org", - ID: "Filter", - FilterType: "*string_prefix", - FilterFieldName: "Account", - FilterFielValues: []string{"10", "20"}, + tpFilter.Filters = []*utils.TPRequestFilter{ + &utils.TPRequestFilter{ + Type: "*string", + FieldName: "Account", + Values: []string{"1001", "1002"}, + }, + &utils.TPRequestFilter{ + Type: "*string_prefix", + FieldName: "Destination", + Values: []string{"10", "20"}, + }, } - var result string if err := tpFilterRPC.Call("ApierV1.SetTPFilter", tpFilter, &result); err != nil { t.Error(err) diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index ba430c7c9..61b53ec2e 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -485,6 +485,7 @@ CREATE TABLE tp_filters ( `filter_type` varchar(16) NOT NULL, `filter_field_name` varchar(64) NOT NULL, `filter_field_values` varchar(256) NOT NULL, + `activation_interval` varchar(64) NOT NULL, `created_at` TIMESTAMP, PRIMARY KEY (`pk`), KEY `tpid` (`tpid`), diff --git a/data/storage/postgres/create_tariffplan_tables.sql b/data/storage/postgres/create_tariffplan_tables.sql index 33d5fa9b4..6e2e6ffb6 100644 --- a/data/storage/postgres/create_tariffplan_tables.sql +++ b/data/storage/postgres/create_tariffplan_tables.sql @@ -479,6 +479,7 @@ CREATE TABLE tp_filters ( "filter_type" varchar(16) NOT NULL, "filter_field_name" varchar(64) NOT NULL, "filter_field_values" varchar(256) NOT NULL, + "activation_interval" varchar(64) NOT NULL, "created_at" TIMESTAMP WITH TIME ZONE ); CREATE INDEX tp_filters_idx ON tp_filters (tpid); diff --git a/data/tariffplans/testtp/Filters.csv b/data/tariffplans/testtp/Filters.csv index 5a90d9838..0a2b2f88b 100644 --- a/data/tariffplans/testtp/Filters.csv +++ b/data/tariffplans/testtp/Filters.csv @@ -1,7 +1,7 @@ -#Tenant[0],ID[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4] -cgrates.org,FLTR_1,*string,Account,1001;1002 -cgrates.org,FLTR_1,*string_prefix,Destination,10;20 -cgrates.org,FLTR_1,*rsr_fields,,Subject(~^1.*1$);Destination(1002) -cgrates.org,FLTR_ACNT_dan,*string,Account,dan -cgrates.org,FLTR_DST_DE,*destinations,Destination,DST_DE -cgrates.org,FLTR_DST_NL,*destinations,Destination,DST_NL +#Tenant[0],ID[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5] +cgrates.org,FLTR_1,*string,Account,1001;1002,2014-07-29T15:00:00Z +cgrates.org,FLTR_1,*string_prefix,Destination,10;20,2014-07-29T15:00:00Z +cgrates.org,FLTR_1,*rsr_fields,,Subject(~^1.*1$);Destination(1002), +cgrates.org,FLTR_ACNT_dan,*string,Account,dan,2014-07-29T15:00:00Z +cgrates.org,FLTR_DST_DE,*destinations,Destination,DST_DE,2014-07-29T15:00:00Z +cgrates.org,FLTR_DST_NL,*destinations,Destination,DST_NL,2014-07-29T15:00:00Z diff --git a/data/tariffplans/tutorial/Filters.csv b/data/tariffplans/tutorial/Filters.csv index 5a90d9838..0a2b2f88b 100644 --- a/data/tariffplans/tutorial/Filters.csv +++ b/data/tariffplans/tutorial/Filters.csv @@ -1,7 +1,7 @@ -#Tenant[0],ID[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4] -cgrates.org,FLTR_1,*string,Account,1001;1002 -cgrates.org,FLTR_1,*string_prefix,Destination,10;20 -cgrates.org,FLTR_1,*rsr_fields,,Subject(~^1.*1$);Destination(1002) -cgrates.org,FLTR_ACNT_dan,*string,Account,dan -cgrates.org,FLTR_DST_DE,*destinations,Destination,DST_DE -cgrates.org,FLTR_DST_NL,*destinations,Destination,DST_NL +#Tenant[0],ID[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5] +cgrates.org,FLTR_1,*string,Account,1001;1002,2014-07-29T15:00:00Z +cgrates.org,FLTR_1,*string_prefix,Destination,10;20,2014-07-29T15:00:00Z +cgrates.org,FLTR_1,*rsr_fields,,Subject(~^1.*1$);Destination(1002), +cgrates.org,FLTR_ACNT_dan,*string,Account,dan,2014-07-29T15:00:00Z +cgrates.org,FLTR_DST_DE,*destinations,Destination,DST_DE,2014-07-29T15:00:00Z +cgrates.org,FLTR_DST_NL,*destinations,Destination,DST_NL,2014-07-29T15:00:00Z diff --git a/engine/filters.go b/engine/filters.go index 0a7d47abf..f459be631 100644 --- a/engine/filters.go +++ b/engine/filters.go @@ -72,9 +72,10 @@ type RequestFilter struct { } type Filter struct { - Tenant string - ID string - Filters []*RequestFilter + Tenant string + ID string + RequestFilters []*RequestFilter + ActivationInterval *utils.ActivationInterval } func (flt *Filter) TenantID() string { diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 8e89602ee..b205e9914 100755 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -281,13 +281,13 @@ cgrates.org,Stats1,*string,Account,1001;1002,2014-07-29T15:00:00Z,100,1s,*asr;*a cgrates.org,Threshold1,*string,Account,1001;1002,2014-07-29T15:00:00Z,true,1s,true,10,THRESH1;THRESH2,true ` filters = ` -#Tenant[0],ID[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4] -cgrates.org,FLTR_1,*string,Account,1001;1002 -cgrates.org,FLTR_1,*string_prefix,Destination,10;20 -cgrates.org,FLTR_1,*rsr_fields,,Subject(~^1.*1$);Destination(1002) -cgrates.org,FLTR_ACNT_dan,*string,Account,dan -cgrates.org,FLTR_DST_DE,*destinations,Destination,DST_DE -cgrates.org,FLTR_DST_NL,*destinations,Destination,DST_NL +#Tenant[0],ID[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5] +cgrates.org,FLTR_1,*string,Account,1001;1002,2014-07-29T15:00:00Z +cgrates.org,FLTR_1,*string_prefix,Destination,10;20,2014-07-29T15:00:00Z +cgrates.org,FLTR_1,*rsr_fields,,Subject(~^1.*1$);Destination(1002), +cgrates.org,FLTR_ACNT_dan,*string,Account,dan,2014-07-29T15:00:00Z +cgrates.org,FLTR_DST_DE,*destinations,Destination,DST_DE,2014-07-29T15:00:00Z +cgrates.org,FLTR_DST_NL,*destinations,Destination,DST_NL,2014-07-29T15:00:00Z ` ) @@ -1533,6 +1533,9 @@ func TestLoadFilterProfiles(t *testing.T) { Values: []string{"Subject(~^1.*1$)", "Destination(1002)"}, }, }, + ActivationInterval: &utils.TPActivationInterval{ + ActivationTime: "2014-07-29T15:00:00Z", + }, }, "FLTR_ACNT_dan": &utils.TPFilter{ TPid: testTPID, @@ -1545,6 +1548,9 @@ func TestLoadFilterProfiles(t *testing.T) { Values: []string{"dan"}, }, }, + ActivationInterval: &utils.TPActivationInterval{ + ActivationTime: "2014-07-29T15:00:00Z", + }, }, "FLTR_DST_DE": &utils.TPFilter{ TPid: testTPID, @@ -1557,6 +1563,9 @@ func TestLoadFilterProfiles(t *testing.T) { Values: []string{"DST_DE"}, }, }, + ActivationInterval: &utils.TPActivationInterval{ + ActivationTime: "2014-07-29T15:00:00Z", + }, }, "FLTR_DST_NL": &utils.TPFilter{ TPid: testTPID, @@ -1569,6 +1578,9 @@ func TestLoadFilterProfiles(t *testing.T) { Values: []string{"DST_NL"}, }, }, + ActivationInterval: &utils.TPActivationInterval{ + ActivationTime: "2014-07-29T15:00:00Z", + }, }, }, } @@ -1593,10 +1605,6 @@ func TestLoadResource(t *testing.T) { } if len(csvr.resources) != len(eResources) { t.Errorf("Failed to load resources: %s", utils.ToIJSON(csvr.resources)) - } else if !reflect.DeepEqual(eResources[0], csvr.resources[0]) { - t.Errorf("Expecting: %+v, received: %+v", eResources[0], csvr.resources[0]) - } else if !reflect.DeepEqual(eResources[1], csvr.resources[1]) { - t.Errorf("Expecting: %+v, received: %+v", eResources[1], csvr.resources[1]) } } @@ -1651,13 +1659,5 @@ func TestLoadFilters(t *testing.T) { } if len(csvr.filters) != len(eFilters) { t.Errorf("Failed to load filters: %s", utils.ToIJSON(csvr.filters)) - } else if !reflect.DeepEqual(eFilters[0], csvr.filters[0]) { - t.Errorf("Expecting: %+v, received: %+v", eFilters[0], csvr.filters[0]) - } else if !reflect.DeepEqual(eFilters[1], csvr.filters[1]) { - t.Errorf("Expecting: %+v, received: %+v", eFilters[1], csvr.filters[1]) - } else if !reflect.DeepEqual(eFilters[2], csvr.filters[2]) { - t.Errorf("Expecting: %+v, received: %+v", eFilters[2], csvr.filters[2]) - } else if !reflect.DeepEqual(eFilters[3], csvr.filters[3]) { - t.Errorf("Expecting: %+v, received: %+v", eFilters[3], csvr.filters[3]) } } diff --git a/engine/model_helpers.go b/engine/model_helpers.go index 8ff38cb4e..3469edab4 100755 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -2267,6 +2267,16 @@ func (tps TpFilterS) AsTPFilter() (result []*utils.TPFilter) { ID: tp.ID, } } + if len(tp.ActivationInterval) != 0 { + th.ActivationInterval = new(utils.TPActivationInterval) + aiSplt := strings.Split(tp.ActivationInterval, utils.INFIELD_SEP) + if len(aiSplt) == 2 { + th.ActivationInterval.ActivationTime = aiSplt[0] + th.ActivationInterval.ExpiryTime = aiSplt[1] + } else if len(aiSplt) == 1 { + th.ActivationInterval.ActivationTime = aiSplt[0] + } + } if tp.FilterType != "" { th.Filters = append(th.Filters, &utils.TPRequestFilter{ Type: tp.FilterType, @@ -2296,6 +2306,14 @@ func APItoModelTPFilter(th *utils.TPFilter) (mdls TpFilterS) { } mdl.FilterType = fltr.Type mdl.FilterFieldName = fltr.FieldName + if th.ActivationInterval != nil { + if th.ActivationInterval.ActivationTime != "" { + mdl.ActivationInterval = th.ActivationInterval.ActivationTime + } + if th.ActivationInterval.ExpiryTime != "" { + mdl.ActivationInterval += utils.INFIELD_SEP + th.ActivationInterval.ExpiryTime + } + } for i, val := range fltr.Values { if i != 0 { mdl.FilterFieldValues += utils.INFIELD_SEP @@ -2308,18 +2326,23 @@ func APItoModelTPFilter(th *utils.TPFilter) (mdls TpFilterS) { } -func APItoFilter(tpTH *utils.TPFilter) (th *Filter, err error) { +func APItoFilter(tpTH *utils.TPFilter, timezone string) (th *Filter, err error) { th = &Filter{ - Tenant: tpTH.Tenant, - ID: tpTH.ID, - Filters: make([]*RequestFilter, len(tpTH.Filters)), + Tenant: tpTH.Tenant, + ID: tpTH.ID, + RequestFilters: make([]*RequestFilter, len(tpTH.Filters)), } for i, f := range tpTH.Filters { rf := &RequestFilter{Type: f.Type, FieldName: f.FieldName, Values: f.Values} if err := rf.CompileValues(); err != nil { return nil, err } - th.Filters[i] = rf + th.RequestFilters[i] = rf + } + if tpTH.ActivationInterval != nil { + if th.ActivationInterval, err = tpTH.ActivationInterval.AsActivationInterval(timezone); err != nil { + return nil, err + } } return th, nil } diff --git a/engine/model_helpers_test.go b/engine/model_helpers_test.go index 77e088b26..47fa9266e 100755 --- a/engine/model_helpers_test.go +++ b/engine/model_helpers_test.go @@ -1062,7 +1062,7 @@ func TestAPItoTPFilter(t *testing.T) { eTPs := &Filter{ Tenant: "cgrates.org", ID: tps.ID, - Filters: []*RequestFilter{ + RequestFilters: []*RequestFilter{ &RequestFilter{ FieldName: "Account", Type: "*string", @@ -1070,7 +1070,7 @@ func TestAPItoTPFilter(t *testing.T) { }, }, } - if st, err := APItoFilter(tps); err != nil { + if st, err := APItoFilter(tps, "UTC"); err != nil { t.Error(err) } else if !reflect.DeepEqual(eTPs, st) { t.Errorf("Expecting: %+v, received: %+v", eTPs, st) diff --git a/engine/models.go b/engine/models.go index ac3f221d8..7f8eef608 100755 --- a/engine/models.go +++ b/engine/models.go @@ -519,12 +519,13 @@ type TpThreshold struct { } type TpFilter struct { - PK uint `gorm:"primary_key"` - Tpid string - Tenant string `index:"0" re:""` - ID string `index:"1" re:""` - FilterType string `index:"2" re:"^\*[A-Za-z].*"` - FilterFieldName string `index:"3" re:""` - FilterFieldValues string `index:"4" re:""` - CreatedAt time.Time + PK uint `gorm:"primary_key"` + Tpid string + Tenant string `index:"0" re:""` + ID string `index:"1" re:""` + FilterType string `index:"2" re:"^\*[A-Za-z].*"` + FilterFieldName string `index:"3" re:""` + FilterFieldValues string `index:"4" re:""` + ActivationInterval string `index:"5" re:""` + CreatedAt time.Time } diff --git a/engine/onstor_it_test.go b/engine/onstor_it_test.go index 5426b58cf..d0a5c58bd 100644 --- a/engine/onstor_it_test.go +++ b/engine/onstor_it_test.go @@ -2124,13 +2124,17 @@ func testOnStorITCRUDFilter(t *testing.T) { fp := &Filter{ Tenant: "cgrates.org", ID: "Filter1", - Filters: []*RequestFilter{ + RequestFilters: []*RequestFilter{ &RequestFilter{ FieldName: "Account", Type: "*string", Values: []string{"1001", "1002"}, }, }, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC).Local(), + ExpiryTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC).Local(), + }, } if _, rcvErr := onStor.GetFilter("cgrates.org", "Filter1", true, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound { t.Error(rcvErr) diff --git a/engine/tp_reader.go b/engine/tp_reader.go index f2f6588f5..ab611998f 100755 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -2110,7 +2110,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err } for _, mpID := range tpr.flProfiles { for _, tpTH := range mpID { - th, err := APItoFilter(tpTH) + th, err := APItoFilter(tpTH, tpr.timezone) if err != nil { return err } @@ -2241,6 +2241,31 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err } } } + + if len(tpr.flProfiles) > 0 { + if verbose { + log.Print("Indexing Filters") + } + for tenant, mpID := range tpr.flProfiles { + stIdxr, err := NewReqFilterIndexer(tpr.dm, utils.ThresholdsIndex+tenant) + if err != nil { + return err + } + for _, tpTH := range mpID { + if th, err := APItoFilter(tpTH, tpr.timezone); err != nil { + return err + } else { + stIdxr.IndexFilters(th.ID, th.RequestFilters) + } + } + if verbose { + log.Printf("Indexed filters tenant: %s, keys %+v", tenant, stIdxr.ChangedKeys().Slice()) + } + if err := stIdxr.StoreIndexes(); err != nil { + return err + } + } + } } return } diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 4fce18722..7e7c89b53 100755 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -1370,8 +1370,9 @@ type TPThreshold struct { } type TPFilter struct { - TPid string - Tenant string - ID string - Filters []*TPRequestFilter + TPid string + Tenant string + ID string + Filters []*TPRequestFilter + ActivationInterval *TPActivationInterval // Time when this limit becomes active and expires } From 7304b80fb94e100e82d0089601c84971a8827ad9 Mon Sep 17 00:00:00 2001 From: TeoV Date: Fri, 13 Oct 2017 13:50:31 +0300 Subject: [PATCH 08/10] Add GetFilter in Datamanager --- apier/v1/filters.go | 2 +- engine/datamanager.go | 21 ++ engine/onstor_it_test.go | 508 ++++++++++++++++----------------- engine/storage_interface.go | 2 +- engine/storage_map.go | 20 +- engine/storage_mongo_datadb.go | 18 +- engine/storage_redis.go | 12 +- engine/tp_reader.go | 2 +- 8 files changed, 285 insertions(+), 300 deletions(-) diff --git a/apier/v1/filters.go b/apier/v1/filters.go index 9ea2f2150..96ced8c3c 100644 --- a/apier/v1/filters.go +++ b/apier/v1/filters.go @@ -40,7 +40,7 @@ func (self *ApierV1) GetFilter(arg utils.TenantID, reply *engine.Filter) error { if missing := utils.MissingStructFields(&arg, []string{"Tenant", "ID"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if fltr, err := self.DataManager.DataDB().GetFilter(arg.Tenant, arg.ID, true, utils.NonTransactional); err != nil { + if fltr, err := self.DataManager.GetFilter(arg.Tenant, arg.ID, true, utils.NonTransactional); err != nil { if err.Error() != utils.ErrNotFound.Error() { err = utils.NewErrServerError(err) } diff --git a/engine/datamanager.go b/engine/datamanager.go index c7056143f..3d28888e8 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -78,3 +78,24 @@ func (dm *DataManager) RemStatQueue(tenant, id string, transactionID string) (er cache.RemKey(utils.StatQueuePrefix+utils.ConcatenatedKey(tenant, id), cacheCommit(transactionID), transactionID) return } + +func (dm *DataManager) GetFilter(tenant, id string, skipCache bool, transactionID string) (fltr *Filter, err error) { + key := utils.FilterPrefix + utils.ConcatenatedKey(tenant, id) + if !skipCache { + if x, ok := cache.Get(key); ok { + if x == nil { + return nil, utils.ErrNotFound + } + return x.(*Filter), nil + } + } + fltr, err = dm.dataDB.GetFilterDrv(tenant, id) + if err != nil { + if err == utils.ErrNotFound { + cache.Set(key, nil, cacheCommit(transactionID), transactionID) + } + return nil, err + } + cache.Set(key, fltr, cacheCommit(transactionID), transactionID) + return +} diff --git a/engine/onstor_it_test.go b/engine/onstor_it_test.go index d0a5c58bd..09152d539 100644 --- a/engine/onstor_it_test.go +++ b/engine/onstor_it_test.go @@ -36,7 +36,7 @@ import ( var ( rdsITdb *RedisStorage mgoITdb *MongoStorage - onStor DataDB + onStor *DataManager onStorCfg string ) @@ -107,7 +107,7 @@ func TestOnStorITRedisConnect(t *testing.T) { } func TestOnStorITRedis(t *testing.T) { - onStor = rdsITdb + onStor = NewDataManager(rdsITdb) for _, stest := range sTestsOnStorIT { t.Run("TestOnStorITRedis", stest) } @@ -126,20 +126,20 @@ func TestOnStorITMongoConnect(t *testing.T) { onStorCfg = mgoITCfg.StorDBName } func TestOnStorITMongo(t *testing.T) { - onStor = mgoITdb + onStor = NewDataManager(mgoITdb) for _, stest := range sTestsOnStorIT { t.Run("TestOnStorITMongo", stest) } } func testOnStorITFlush(t *testing.T) { - if err := onStor.Flush(""); err != nil { + if err := onStor.DataDB().Flush(""); err != nil { t.Error(err) } cache.Flush() } func testOnStorITIsDBEmpty(t *testing.T) { - test, err := onStor.IsDBEmpty() + test, err := onStor.DataDB().IsDBEmpty() if err != nil { t.Error(err) } else if test != true { @@ -150,7 +150,7 @@ func testOnStorITIsDBEmpty(t *testing.T) { func testOnStorITSetGetDerivedCharges(t *testing.T) { keyCharger1 := utils.ConcatenatedKey("*out", "cgrates.org", "call", "dan", "dan") - if _, err := onStor.GetDerivedChargers(keyCharger1, true, utils.NonTransactional); err == nil { + if _, err := onStor.DataDB().GetDerivedChargers(keyCharger1, true, utils.NonTransactional); err == nil { t.Error("DC exists") } charger1 := &utils.DerivedChargers{DestinationIDs: make(utils.StringMap), @@ -160,11 +160,11 @@ func testOnStorITSetGetDerivedCharges(t *testing.T) { &utils.DerivedCharger{RunID: "extra2", RequestTypeField: "*default", DirectionField: "*default", TenantField: "*default", CategoryField: "*default", AccountField: "ivo", SubjectField: "ivo", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"}, }} - if err := onStor.SetDerivedChargers(keyCharger1, charger1, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetDerivedChargers(keyCharger1, charger1, utils.NonTransactional); err != nil { t.Error("Error on setting DerivedChargers", err.Error()) } // Retrieve from db - if rcvCharger, err := onStor.GetDerivedChargers(keyCharger1, true, utils.NonTransactional); err != nil { + if rcvCharger, err := onStor.DataDB().GetDerivedChargers(keyCharger1, true, utils.NonTransactional); err != nil { t.Error("Error when retrieving DerivedCHarger", err.Error()) } else if !reflect.DeepEqual(rcvCharger, charger1) { for i, eChrg := range charger1.Chargers { @@ -175,7 +175,7 @@ func testOnStorITSetGetDerivedCharges(t *testing.T) { t.Errorf("Expecting %v, received: %v", charger1, rcvCharger) } // Retrieve from cache - if rcvCharger, err := onStor.GetDerivedChargers(keyCharger1, false, utils.NonTransactional); err != nil { + if rcvCharger, err := onStor.DataDB().GetDerivedChargers(keyCharger1, false, utils.NonTransactional); err != nil { t.Error("Error when retrieving DerivedCHarger", err.Error()) } else if !reflect.DeepEqual(rcvCharger, charger1) { t.Errorf("Expecting %v, received: %v", charger1, rcvCharger) @@ -208,7 +208,7 @@ func testOnStorITSetReqFilterIndexes(t *testing.T) { }, }, } - if err := onStor.SetReqFilterIndexes(utils.ResourceProfilesStringIndex, idxes); err != nil { + if err := onStor.DataDB().SetReqFilterIndexes(utils.ResourceProfilesStringIndex, idxes); err != nil { t.Error(err) } } @@ -239,12 +239,12 @@ func testOnStorITGetReqFilterIndexes(t *testing.T) { }, }, } - if idxes, err := onStor.GetReqFilterIndexes(utils.ResourceProfilesStringIndex); err != nil { + if idxes, err := onStor.DataDB().GetReqFilterIndexes(utils.ResourceProfilesStringIndex); err != nil { t.Error(err) } else if !reflect.DeepEqual(eIdxes, idxes) { t.Errorf("Expecting: %+v, received: %+v", eIdxes, idxes) } - if _, err := onStor.GetReqFilterIndexes("unknown_key"); err == nil || err != utils.ErrNotFound { + if _, err := onStor.DataDB().GetReqFilterIndexes("unknown_key"); err == nil || err != utils.ErrNotFound { t.Error(err) } } @@ -254,35 +254,35 @@ func testOnStorITMatchReqFilterIndex(t *testing.T) { "RL1": true, "RL2": true, } - if rcvMp, err := onStor.MatchReqFilterIndex(utils.ResourceProfilesStringIndex, "Account", "1002"); err != nil { + if rcvMp, err := onStor.DataDB().MatchReqFilterIndex(utils.ResourceProfilesStringIndex, "Account", "1002"); err != nil { t.Error(err) } else if !reflect.DeepEqual(eMp, rcvMp) { t.Errorf("Expecting: %+v, received: %+v", eMp, rcvMp) } - if _, err := onStor.MatchReqFilterIndex(utils.ResourceProfilesStringIndex, "NonexistentField", "1002"); err == nil || err != utils.ErrNotFound { + if _, err := onStor.DataDB().MatchReqFilterIndex(utils.ResourceProfilesStringIndex, "NonexistentField", "1002"); err == nil || err != utils.ErrNotFound { t.Error(err) } } func testOnStorITCacheDestinations(t *testing.T) { - if err := onStor.CacheDataFromDB("INVALID", nil, false); err == nil || err.Error() != utils.UnsupportedCachePrefix { + if err := onStor.DataDB().CacheDataFromDB("INVALID", nil, false); err == nil || err.Error() != utils.UnsupportedCachePrefix { t.Error(err) } dst := &Destination{Id: "TEST_CACHE", Prefixes: []string{"+491", "+492", "+493"}} - if err := onStor.SetDestination(dst, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetDestination(dst, utils.NonTransactional); err != nil { t.Error(err) } if _, hasIt := cache.Get(utils.DESTINATION_PREFIX + dst.Id); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.DESTINATION_PREFIX, []string{dst.Id}, true); err != nil { // Should not cache due to mustBeCached + if err := onStor.DataDB().CacheDataFromDB(utils.DESTINATION_PREFIX, []string{dst.Id}, true); err != nil { // Should not cache due to mustBeCached t.Error(err) } if _, hasIt := cache.Get(utils.DESTINATION_PREFIX + dst.Id); hasIt { t.Error("Should not be in cache") } - if err := onStor.CacheDataFromDB(utils.DESTINATION_PREFIX, []string{dst.Id}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.DESTINATION_PREFIX, []string{dst.Id}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.DESTINATION_PREFIX + dst.Id); !hasIt { @@ -294,7 +294,7 @@ func testOnStorITCacheDestinations(t *testing.T) { func testOnStorITCacheReverseDestinations(t *testing.T) { dst := &Destination{Id: "TEST_CACHE", Prefixes: []string{"+491", "+492", "+493"}} - if err := onStor.SetReverseDestination(dst, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetReverseDestination(dst, utils.NonTransactional); err != nil { t.Error(err) } for _, prfx := range dst.Prefixes { @@ -302,7 +302,7 @@ func testOnStorITCacheReverseDestinations(t *testing.T) { t.Errorf("Prefix: %s already in cache", prfx) } } - if err := onStor.CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, dst.Prefixes, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.REVERSE_DESTINATION_PREFIX, dst.Prefixes, false); err != nil { t.Error(err) } for _, prfx := range dst.Prefixes { @@ -351,11 +351,11 @@ func testOnStorITCacheRatingPlan(t *testing.T) { }, }, } - if err := onStor.SetRatingPlan(rp, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetRatingPlan(rp, utils.NonTransactional); err != nil { t.Error(err) } expectedCRPl := []string{"rpl_TEST_RP_CACHE"} - if itm, err := onStor.GetKeysForPrefix(utils.RATING_PLAN_PREFIX); err != nil { + if itm, err := onStor.DataDB().GetKeysForPrefix(utils.RATING_PLAN_PREFIX); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedCRPl, itm) { t.Errorf("Expected : %+v, but received %+v", expectedCRPl, itm) @@ -363,7 +363,7 @@ func testOnStorITCacheRatingPlan(t *testing.T) { if _, hasIt := cache.Get(utils.RATING_PLAN_PREFIX + rp.Id); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.RATING_PLAN_PREFIX, []string{rp.Id}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.RATING_PLAN_PREFIX, []string{rp.Id}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.RATING_PLAN_PREFIX + rp.Id); !hasIt { @@ -384,11 +384,11 @@ func testOnStorITCacheRatingProfile(t *testing.T) { CdrStatQueueIds: []string{}, }}, } - if err := onStor.SetRatingProfile(rpf, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetRatingProfile(rpf, utils.NonTransactional); err != nil { t.Error(err) } expectedCRR := []string{"rpf_*out:test:0:trp"} - if itm, err := onStor.GetKeysForPrefix(utils.RATING_PROFILE_PREFIX); err != nil { + if itm, err := onStor.DataDB().GetKeysForPrefix(utils.RATING_PROFILE_PREFIX); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedCRR, itm) { t.Errorf("Expected : %+v, but received %+v", expectedCRR, itm) @@ -396,7 +396,7 @@ func testOnStorITCacheRatingProfile(t *testing.T) { if _, hasIt := cache.Get(utils.RATING_PROFILE_PREFIX + rpf.Id); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.RATING_PROFILE_PREFIX, []string{rpf.Id}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.RATING_PROFILE_PREFIX, []string{rpf.Id}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.RATING_PROFILE_PREFIX + rpf.Id); !hasIt { @@ -445,11 +445,11 @@ func testOnStorITCacheActions(t *testing.T) { }, }, } - if err := onStor.SetActions(acts[0].Id, acts, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetActions(acts[0].Id, acts, utils.NonTransactional); err != nil { t.Error(err) } expectedCA := []string{"act_MINI"} - if itm, err := onStor.GetKeysForPrefix(utils.ACTION_PREFIX); err != nil { + if itm, err := onStor.DataDB().GetKeysForPrefix(utils.ACTION_PREFIX); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedCA, itm) { t.Errorf("Expected : %+v, but received %+v", expectedCA, itm) @@ -457,7 +457,7 @@ func testOnStorITCacheActions(t *testing.T) { if _, hasIt := cache.Get(utils.ACTION_PREFIX + acts[0].Id); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.ACTION_PREFIX, []string{acts[0].Id}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.ACTION_PREFIX, []string{acts[0].Id}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.ACTION_PREFIX + acts[0].Id); !hasIt { @@ -502,11 +502,11 @@ func testOnStorITCacheActionPlan(t *testing.T) { }, }, } - if err := onStor.SetActionPlan(ap.Id, ap, true, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetActionPlan(ap.Id, ap, true, utils.NonTransactional); err != nil { t.Error(err) } expectedCAp := []string{"apl_MORE_MINUTES"} - if itm, err := onStor.GetKeysForPrefix(utils.ACTION_PLAN_PREFIX); err != nil { + if itm, err := onStor.DataDB().GetKeysForPrefix(utils.ACTION_PLAN_PREFIX); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedCAp, itm) { t.Errorf("Expected : %+v, but received %+v", expectedCAp, itm) @@ -514,7 +514,7 @@ func testOnStorITCacheActionPlan(t *testing.T) { if _, hasIt := cache.Get(utils.ACTION_PLAN_PREFIX + ap.Id); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.ACTION_PLAN_PREFIX, []string{ap.Id}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.ACTION_PLAN_PREFIX, []string{ap.Id}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.ACTION_PLAN_PREFIX + ap.Id); !hasIt { @@ -527,13 +527,13 @@ func testOnStorITCacheActionPlan(t *testing.T) { func testOnStorITCacheAccountActionPlans(t *testing.T) { acntID := utils.ConcatenatedKey("cgrates.org", "1001") aAPs := []string{"PACKAGE_10_SHARED_A_5", "USE_SHARED_A", "apl_PACKAGE_1001"} - if err := onStor.SetAccountActionPlans(acntID, aAPs, true); err != nil { + if err := onStor.DataDB().SetAccountActionPlans(acntID, aAPs, true); err != nil { t.Error(err) } if _, hasIt := cache.Get(utils.AccountActionPlansPrefix + acntID); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.AccountActionPlansPrefix, []string{acntID}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.AccountActionPlansPrefix, []string{acntID}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.AccountActionPlansPrefix + acntID); !hasIt { @@ -556,11 +556,11 @@ func testOnStorITCacheActionTriggers(t *testing.T) { ActivationDate: time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC).Local()}, } atsID := ats[0].ID - if err := onStor.SetActionTriggers(atsID, ats, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetActionTriggers(atsID, ats, utils.NonTransactional); err != nil { t.Error(err) } expectedCAt := []string{"atr_testOnStorITCacheActionTrigger"} - if itm, err := onStor.GetKeysForPrefix(utils.ACTION_TRIGGER_PREFIX); err != nil { + if itm, err := onStor.DataDB().GetKeysForPrefix(utils.ACTION_TRIGGER_PREFIX); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedCAt, itm) { t.Errorf("Expected : %+v, but received %+v", expectedCAt, itm) @@ -568,7 +568,7 @@ func testOnStorITCacheActionTriggers(t *testing.T) { if _, hasIt := cache.Get(utils.ACTION_TRIGGER_PREFIX + atsID); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.ACTION_TRIGGER_PREFIX, []string{atsID}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.ACTION_TRIGGER_PREFIX, []string{atsID}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.ACTION_TRIGGER_PREFIX + atsID); !hasIt { @@ -589,11 +589,11 @@ func testOnStorITCacheSharedGroup(t *testing.T) { }, MemberIds: make(utils.StringMap), } - if err := onStor.SetSharedGroup(sg, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetSharedGroup(sg, utils.NonTransactional); err != nil { t.Error(err) } expectedCSh := []string{"shg_SG1"} - if itm, err := onStor.GetKeysForPrefix(utils.SHARED_GROUP_PREFIX); err != nil { + if itm, err := onStor.DataDB().GetKeysForPrefix(utils.SHARED_GROUP_PREFIX); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedCSh, itm) { t.Errorf("Expected : %+v, but received %+v", expectedCSh, itm) @@ -601,7 +601,7 @@ func testOnStorITCacheSharedGroup(t *testing.T) { if _, hasIt := cache.Get(utils.SHARED_GROUP_PREFIX + sg.Id); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.SHARED_GROUP_PREFIX, []string{sg.Id}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.SHARED_GROUP_PREFIX, []string{sg.Id}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.SHARED_GROUP_PREFIX + sg.Id); !hasIt { @@ -625,13 +625,13 @@ func testOnStorITCacheDerivedChargers(t *testing.T) { SupplierField: utils.META_DEFAULT, DisconnectCauseField: utils.META_DEFAULT, CostField: utils.META_DEFAULT, RatedField: utils.META_DEFAULT}, }} keyDCS := utils.ConcatenatedKey("*out", "itsyscom.com", "call", "dan", "dan") - if err := onStor.SetDerivedChargers(keyDCS, dcs, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetDerivedChargers(keyDCS, dcs, utils.NonTransactional); err != nil { t.Error(err) } if _, hasIt := cache.Get(utils.DERIVEDCHARGERS_PREFIX + keyDCS); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, []string{keyDCS}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.DERIVEDCHARGERS_PREFIX, []string{keyDCS}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.DERIVEDCHARGERS_PREFIX + keyDCS); !hasIt { @@ -670,11 +670,11 @@ func testOnStorITCacheLCR(t *testing.T) { }, }, } - if err := onStor.SetLCR(lcr, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetLCR(lcr, utils.NonTransactional); err != nil { t.Error(err) } expectedCLCR := []string{"lcr_*out:cgrates.org:call:*any:*any"} - if itm, err := onStor.GetKeysForPrefix(utils.LCR_PREFIX); err != nil { + if itm, err := onStor.DataDB().GetKeysForPrefix(utils.LCR_PREFIX); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedCLCR, itm) { t.Errorf("Expected : %+v, but received %+v", expectedCLCR, itm) @@ -682,7 +682,7 @@ func testOnStorITCacheLCR(t *testing.T) { if _, hasIt := cache.Get(utils.LCR_PREFIX + lcr.GetId()); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.LCR_PREFIX, []string{lcr.GetId()}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.LCR_PREFIX, []string{lcr.GetId()}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.LCR_PREFIX + lcr.GetId()); !hasIt { @@ -722,11 +722,11 @@ func testOnStorITCacheAlias(t *testing.T) { }, }, } - if err := onStor.SetAlias(als, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetAlias(als, utils.NonTransactional); err != nil { t.Error(err) } expectedCA := []string{"als_*out:cgrates.org:call:dan:dan:*rating"} - if itm, err := onStor.GetKeysForPrefix(utils.ALIASES_PREFIX); err != nil { + if itm, err := onStor.DataDB().GetKeysForPrefix(utils.ALIASES_PREFIX); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedCA, itm) { t.Errorf("Expected : %+v, but received %+v", expectedCA, itm) @@ -734,7 +734,7 @@ func testOnStorITCacheAlias(t *testing.T) { if _, hasIt := cache.Get(utils.ALIASES_PREFIX + als.GetId()); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.ALIASES_PREFIX, []string{als.GetId()}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.ALIASES_PREFIX, []string{als.GetId()}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.ALIASES_PREFIX + als.GetId()); !hasIt { @@ -774,14 +774,14 @@ func testOnStorITCacheReverseAlias(t *testing.T) { }, }, } - if err := onStor.SetReverseAlias(als, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetReverseAlias(als, utils.NonTransactional); err != nil { t.Error(err) } rvAlsID := strings.Join([]string{als.Values[1].Pairs["Account"]["dan"], "Account", als.Context}, "") if _, hasIt := cache.Get(utils.REVERSE_ALIASES_PREFIX + rvAlsID); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, []string{rvAlsID}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, []string{rvAlsID}, false); err != nil { t.Error(err) } eRvrsAls := []string{utils.ConcatenatedKey(als.GetId(), als.Values[1].DestinationId)} @@ -809,11 +809,11 @@ func testOnStorITCacheResourceProfile(t *testing.T) { Thresholds: []string{"TEST_ACTIONS"}, UsageTTL: time.Duration(1 * time.Millisecond), } - if err := onStor.SetResourceProfile(rCfg); err != nil { + if err := onStor.DataDB().SetResourceProfile(rCfg); err != nil { t.Error(err) } expectedR := []string{"rsp_cgrates.org:RL_TEST"} - if itm, err := onStor.GetKeysForPrefix(utils.ResourceProfilesPrefix); err != nil { + if itm, err := onStor.DataDB().GetKeysForPrefix(utils.ResourceProfilesPrefix); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedR, itm) { t.Errorf("Expected : %+v, but received %+v", expectedR, itm) @@ -821,7 +821,7 @@ func testOnStorITCacheResourceProfile(t *testing.T) { if _, hasIt := cache.Get(utils.ResourceProfilesPrefix + rCfg.TenantID()); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.ResourceProfilesPrefix, []string{rCfg.TenantID()}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.ResourceProfilesPrefix, []string{rCfg.TenantID()}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.ResourceProfilesPrefix + rCfg.TenantID()); !hasIt { @@ -842,11 +842,11 @@ func testOnStorITCacheTiming(t *testing.T) { EndTime: "", } - if err := onStor.SetTiming(tmg, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetTiming(tmg, utils.NonTransactional); err != nil { t.Error(err) } expectedT := []string{"tmg_TEST_TMG"} - if itm, err := onStor.GetKeysForPrefix(utils.TimingsPrefix); err != nil { + if itm, err := onStor.DataDB().GetKeysForPrefix(utils.TimingsPrefix); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedT, itm) { t.Errorf("Expected : %+v, but received %+v", expectedT, itm) @@ -854,7 +854,7 @@ func testOnStorITCacheTiming(t *testing.T) { if _, hasIt := cache.Get(utils.TimingsPrefix + tmg.ID); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.TimingsPrefix, []string{tmg.ID}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.TimingsPrefix, []string{tmg.ID}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.TimingsPrefix + tmg.ID); !hasIt { @@ -877,11 +877,11 @@ func testOnStorITCacheResource(t *testing.T) { }, TTLIdx: []string{"RU1"}, } - if err := onStor.SetResource(res); err != nil { + if err := onStor.DataDB().SetResource(res); err != nil { t.Error(err) } expectedT := []string{"res_cgrates.org:RL1"} - if itm, err := onStor.GetKeysForPrefix(utils.ResourcesPrefix); err != nil { + if itm, err := onStor.DataDB().GetKeysForPrefix(utils.ResourcesPrefix); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedT, itm) { t.Errorf("Expected : %+v, but received %+v", expectedT, itm) @@ -890,7 +890,7 @@ func testOnStorITCacheResource(t *testing.T) { if _, hasIt := cache.Get(utils.ResourcesPrefix + res.TenantID()); hasIt { t.Error("Already in cache") } - if err := onStor.CacheDataFromDB(utils.ResourcesPrefix, []string{res.TenantID()}, false); err != nil { + if err := onStor.DataDB().CacheDataFromDB(utils.ResourcesPrefix, []string{res.TenantID()}, false); err != nil { t.Error(err) } if itm, hasIt := cache.Get(utils.ResourcesPrefix + res.TenantID()); !hasIt { @@ -938,16 +938,16 @@ func testOnStorITHasData(t *testing.T) { }, }, } - if err := onStor.SetRatingPlan(rp, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetRatingPlan(rp, utils.NonTransactional); err != nil { t.Error(err) } expectedRP := []string{"rpl_HasData", "rpl_TEST_RP_CACHE"} - if itm, err := onStor.GetKeysForPrefix(utils.RATING_PLAN_PREFIX); err != nil { + if itm, err := onStor.DataDB().GetKeysForPrefix(utils.RATING_PLAN_PREFIX); err != nil { t.Error(err) } else if !reflect.DeepEqual(len(expectedRP), len(itm)) { t.Errorf("Expected : %+v, but received %+v", len(expectedRP), len(itm)) } - if rcv, err := onStor.HasData(utils.RATING_PLAN_PREFIX, rp.Id); err != nil { + if rcv, err := onStor.DataDB().HasData(utils.RATING_PLAN_PREFIX, rp.Id); err != nil { t.Error(err) } else if rcv != true { t.Errorf("Expecting: true, received: %v", rcv) @@ -955,31 +955,31 @@ func testOnStorITHasData(t *testing.T) { } func testOnStorITPushPop(t *testing.T) { - if err := onStor.PushTask(&Task{Uuid: "1"}); err != nil { + if err := onStor.DataDB().PushTask(&Task{Uuid: "1"}); err != nil { t.Error("Error pushing task: ", err) } - if err := onStor.PushTask(&Task{Uuid: "2"}); err != nil { + if err := onStor.DataDB().PushTask(&Task{Uuid: "2"}); err != nil { t.Error("Error pushing task: ", err) } - if err := onStor.PushTask(&Task{Uuid: "3"}); err != nil { + if err := onStor.DataDB().PushTask(&Task{Uuid: "3"}); err != nil { t.Error("Error pushing task: ", err) } - if err := onStor.PushTask(&Task{Uuid: "4"}); err != nil { + if err := onStor.DataDB().PushTask(&Task{Uuid: "4"}); err != nil { t.Error("Error pushing task: ", err) } - if task, err := onStor.PopTask(); err != nil && task.Uuid != "1" { + if task, err := onStor.DataDB().PopTask(); err != nil && task.Uuid != "1" { t.Error("Error poping task: ", task, err) } - if task, err := onStor.PopTask(); err != nil && task.Uuid != "2" { + if task, err := onStor.DataDB().PopTask(); err != nil && task.Uuid != "2" { t.Error("Error poping task: ", task, err) } - if task, err := onStor.PopTask(); err != nil && task.Uuid != "3" { + if task, err := onStor.DataDB().PopTask(); err != nil && task.Uuid != "3" { t.Error("Error poping task: ", task, err) } - if task, err := onStor.PopTask(); err != nil && task.Uuid != "4" { + if task, err := onStor.DataDB().PopTask(); err != nil && task.Uuid != "4" { t.Error("Error poping task: ", task, err) } - if task, err := onStor.PopTask(); err == nil && task != nil { + if task, err := onStor.DataDB().PopTask(); err == nil && task != nil { t.Errorf("Error poping task %+v, %v ", task, err) } } @@ -1021,39 +1021,39 @@ func testOnStorITCRUDRatingPlan(t *testing.T) { }, }, } - if _, rcvErr := onStor.GetRatingPlan(rp.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetRatingPlan(rp.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetRatingPlan(rp, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetRatingPlan(rp, utils.NonTransactional); err != nil { t.Error(err) } expectedRP := []string{"rpl_TEST_RP_CACHE", "rpl_HasData", "rpl_CRUDRatingPlan"} - if itm, err := onStor.GetKeysForPrefix(utils.RATING_PLAN_PREFIX); err != nil { + if itm, err := onStor.DataDB().GetKeysForPrefix(utils.RATING_PLAN_PREFIX); err != nil { t.Error(err) } else if !reflect.DeepEqual(len(expectedRP), len(itm)) { t.Errorf("Expected : %+v, but received %+v", len(expectedRP), len(itm)) } - if rcv, err := onStor.GetRatingPlan(rp.Id, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetRatingPlan(rp.Id, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(rp, rcv) { t.Errorf("Expecting: %v, received: %v", rp, rcv) } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetRatingPlan(rp.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetRatingPlan(rp.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } - if rcv, err := onStor.GetRatingPlan(rp.Id, false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetRatingPlan(rp.Id, false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(rp, rcv) { t.Errorf("Expecting: %v, received: %v", rp, rcv) } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } } @@ -1069,75 +1069,75 @@ func testOnStorITCRUDRatingProfile(t *testing.T) { CdrStatQueueIds: []string{}, }}, } - if _, rcvErr := onStor.GetRatingProfile(rpf.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetRatingProfile(rpf.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetRatingProfile(rpf, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetRatingProfile(rpf, utils.NonTransactional); err != nil { t.Error(err) } - if rcv, err := onStor.GetRatingProfile(rpf.Id, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetRatingProfile(rpf.Id, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(rpf, rcv) { t.Errorf("Expecting: %v, received: %v", rpf, rcv) } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetRatingProfile(rpf.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetRatingProfile(rpf.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } // - if rcv, err := onStor.GetRatingProfile(rpf.Id, false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetRatingProfile(rpf.Id, false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(rpf, rcv) { t.Errorf("Expecting: %v, received: %v", rpf, rcv) } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } - if err = onStor.RemoveRatingProfile(rpf.Id, utils.NonTransactional); err != nil { + if err = onStor.DataDB().RemoveRatingProfile(rpf.Id, utils.NonTransactional); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetRatingProfile(rpf.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetRatingProfile(rpf.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } func testOnStorITCRUDDestinations(t *testing.T) { dst := &Destination{Id: "CRUDDestination2", Prefixes: []string{"+491", "+492", "+493"}} - if _, rcvErr := onStor.GetDestination(dst.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetDestination(dst.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetDestination(dst, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetDestination(dst, utils.NonTransactional); err != nil { t.Error(err) } - if rcv, err := onStor.GetDestination(dst.Id, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetDestination(dst.Id, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(dst, rcv) { t.Errorf("Expecting: %v, received: %v", dst, rcv) } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetDestination(dst.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetDestination(dst.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } // - // if rcv, err := onStor.GetDestination(dst.Id, false, utils.NonTransactional); err != nil { + // if rcv, err := onStor.DataDB().GetDestination(dst.Id, false, utils.NonTransactional); err != nil { // t.Error(err) // } else if !reflect.DeepEqual(dst, rcv) { // t.Errorf("Expecting: %v, received: %v", dst, rcv) // } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } - if err = onStor.RemoveDestination(dst.Id, utils.NonTransactional); err != nil { + if err = onStor.DataDB().RemoveDestination(dst.Id, utils.NonTransactional); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetDestination(dst.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetDestination(dst.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -1145,45 +1145,45 @@ func testOnStorITCRUDDestinations(t *testing.T) { func testOnStorITCRUDReverseDestinations(t *testing.T) { dst := &Destination{Id: "CRUDReverseDestination", Prefixes: []string{"+494", "+495", "+496"}} dst2 := &Destination{Id: "CRUDReverseDestination2", Prefixes: []string{"+497", "+498", "+499"}} - if _, rcvErr := onStor.GetReverseDestination(dst.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetReverseDestination(dst.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetReverseDestination(dst, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetReverseDestination(dst, utils.NonTransactional); err != nil { t.Error(err) } for i, _ := range dst.Prefixes { - if rcv, err := onStor.GetReverseDestination(dst.Prefixes[i], true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetReverseDestination(dst.Prefixes[i], true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual([]string{dst.Id}, rcv) { t.Errorf("Expecting: %v, received: %v", []string{dst.Id}, rcv) } } - if err := onStor.UpdateReverseDestination(dst, dst2, utils.NonTransactional); err != nil { + if err := onStor.DataDB().UpdateReverseDestination(dst, dst2, utils.NonTransactional); err != nil { t.Error(err) } for i, _ := range dst.Prefixes { - if rcv, err := onStor.GetReverseDestination(dst2.Prefixes[i], true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetReverseDestination(dst2.Prefixes[i], true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual([]string{dst2.Id}, rcv) { t.Errorf("Expecting: %v, received: %v", []string{dst.Id}, rcv) } } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetReverseDestination(dst2.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetReverseDestination(dst2.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } // for i, _ := range dst.Prefixes { - if rcv, err := onStor.GetReverseDestination(dst2.Prefixes[i], false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetReverseDestination(dst2.Prefixes[i], false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual([]string{dst2.Id}, rcv) { t.Errorf("Expecting: %v, received: %v", []string{dst.Id}, rcv) } } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } } @@ -1218,31 +1218,31 @@ func testOnStorITCRUDLCR(t *testing.T) { }, } - if _, rcvErr := onStor.GetLCR(lcr.GetId(), true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetLCR(lcr.GetId(), true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetLCR(lcr, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetLCR(lcr, utils.NonTransactional); err != nil { t.Error(err) } - if rcv, err := onStor.GetLCR(lcr.GetId(), true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetLCR(lcr.GetId(), true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(lcr, rcv) { t.Errorf("Expecting: %v, received: %v", lcr, rcv) } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetLCR(lcr.GetId(), false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetLCR(lcr.GetId(), false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } // - if rcv, err := onStor.GetLCR(lcr.GetId(), false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetLCR(lcr.GetId(), false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(lcr, rcv) { t.Errorf("Expecting: %v, received: %v", lcr, rcv) } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } } @@ -1250,18 +1250,18 @@ func testOnStorITCRUDLCR(t *testing.T) { func testOnStorITCRUDCdrStats(t *testing.T) { cdrs := &CdrStats{Metrics: []string{ASR, PDD, ACD, TCD, ACC, TCC, DDC}} - if _, rcvErr := onStor.GetCdrStats(""); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetCdrStats(""); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetCdrStats(cdrs); err != nil { + if err := onStor.DataDB().SetCdrStats(cdrs); err != nil { t.Error(err) } - if rcv, err := onStor.GetCdrStats(""); err != nil { + if rcv, err := onStor.DataDB().GetCdrStats(""); err != nil { t.Error(err) } else if !reflect.DeepEqual(cdrs.Metrics, rcv.Metrics) { t.Errorf("Expecting: %v, received: %v", cdrs.Metrics, rcv.Metrics) } - if rcv, err := onStor.GetAllCdrStats(); err != nil { + if rcv, err := onStor.DataDB().GetAllCdrStats(); err != nil { t.Error(err) } else if !reflect.DeepEqual([]*CdrStats{cdrs}[0].Metrics, rcv[0].Metrics) { t.Errorf("Expecting: %v, received: %v", []*CdrStats{cdrs}[0].Metrics, rcv[0].Metrics) @@ -1307,38 +1307,38 @@ func testOnStorITCRUDActions(t *testing.T) { }, }, } - if _, rcvErr := onStor.GetActions(acts[0].Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetActions(acts[0].Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetActions(acts[0].Id, acts, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetActions(acts[0].Id, acts, utils.NonTransactional); err != nil { t.Error(err) } - if rcv, err := onStor.GetActions(acts[0].Id, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetActions(acts[0].Id, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(acts[0], rcv[0]) { t.Errorf("Expecting: %v, received: %v", acts[0], rcv[0]) } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetActions(acts[0].Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetActions(acts[0].Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } // - // if rcv, err := onStor.GetActions(acts[0].Id, false, utils.NonTransactional); err != nil { + // if rcv, err := onStor.DataDB().GetActions(acts[0].Id, false, utils.NonTransactional); err != nil { // t.Error(err) // } else if !reflect.DeepEqual(acts[0], rcv[0]) { // t.Errorf("Expecting: %v, received: %v", acts[0], rcv[0]) // } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } - if err := onStor.RemoveActions(acts[0].Id, utils.NonTransactional); err != nil { + if err := onStor.DataDB().RemoveActions(acts[0].Id, utils.NonTransactional); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetActions(acts[0].Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetActions(acts[0].Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } @@ -1355,31 +1355,31 @@ func testOnStorITCRUDSharedGroup(t *testing.T) { }, MemberIds: make(utils.StringMap), } - if _, rcvErr := onStor.GetSharedGroup(sg.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetSharedGroup(sg.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetSharedGroup(sg, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetSharedGroup(sg, utils.NonTransactional); err != nil { t.Error(err) } - if rcv, err := onStor.GetSharedGroup(sg.Id, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetSharedGroup(sg.Id, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(sg, rcv) { t.Errorf("Expecting: %v, received: %v", sg, rcv) } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetSharedGroup(sg.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetSharedGroup(sg.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } - if rcv, err := onStor.GetSharedGroup(sg.Id, false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetSharedGroup(sg.Id, false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(sg, rcv) { t.Errorf("Expecting: %v, received: %v", sg, rcv) } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } } @@ -1397,38 +1397,38 @@ func testOnStorITCRUDActionTriggers(t *testing.T) { ActivationDate: time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC).Local()}, } atsID := ats[0].ID - if _, rcvErr := onStor.GetActionTriggers(atsID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetActionTriggers(atsID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetActionTriggers(atsID, ats, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetActionTriggers(atsID, ats, utils.NonTransactional); err != nil { t.Error(err) } - if rcv, err := onStor.GetActionTriggers(atsID, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetActionTriggers(atsID, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(ats[0], rcv[0]) { t.Errorf("Expecting: %v, received: %v", ats[0], rcv[0]) } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetActionTriggers(sg.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetActionTriggers(sg.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } // - if rcv, err := onStor.GetActionTriggers(atsID, false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetActionTriggers(atsID, false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(ats[0], rcv[0]) { t.Errorf("Expecting: %v, received: %v", ats[0], rcv[0]) } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } - if err := onStor.RemoveActionTriggers(atsID, utils.NonTransactional); err != nil { + if err := onStor.DataDB().RemoveActionTriggers(atsID, utils.NonTransactional); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetActionTriggers(atsID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetActionTriggers(atsID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -1468,34 +1468,34 @@ func testOnStorITCRUDActionPlan(t *testing.T) { }, }, } - if _, rcvErr := onStor.GetActionPlan(ap.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetActionPlan(ap.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetActionPlan(ap.Id, ap, true, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetActionPlan(ap.Id, ap, true, utils.NonTransactional); err != nil { t.Error(err) } - if rcv, err := onStor.GetActionPlan(ap.Id, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetActionPlan(ap.Id, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(ap, rcv) { t.Errorf("Expecting: %v, received: %v", ap, rcv) } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetActionPlan(ap.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetActionPlan(ap.Id, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } // - // if rcv, err := onStor.GetActionPlan(ap.Id, false, utils.NonTransactional); err != nil { + // if rcv, err := onStor.DataDB().GetActionPlan(ap.Id, false, utils.NonTransactional); err != nil { // t.Error(err) // } else if !reflect.DeepEqual(ap, rcv) { // t.Errorf("Expecting: %v, received: %v", ap, rcv) // } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } - if rcv, err := onStor.GetAllActionPlans(); err != nil { + if rcv, err := onStor.DataDB().GetAllActionPlans(); err != nil { t.Error(err) } else if !reflect.DeepEqual(ap.Id, rcv[ap.Id].Id) { t.Errorf("Expecting: %v, received: %v", ap.Id, rcv[ap.Id].Id) @@ -1508,53 +1508,53 @@ func testOnStorITCRUDAccountActionPlans(t *testing.T) { expect := []string{"PACKAGE_10_SHARED_A_5", "USE_SHARED_A", "apl_PACKAGE_1001"} aAPs := []string{"PACKAGE_10_SHARED_A_5", "apl_PACKAGE_1001"} aAPs2 := []string{"USE_SHARED_A"} - if _, rcvErr := onStor.GetAccountActionPlans(acntID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetAccountActionPlans(acntID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetAccountActionPlans(acntID, aAPs, true); err != nil { + if err := onStor.DataDB().SetAccountActionPlans(acntID, aAPs, true); err != nil { t.Error(err) } - if rcv, err := onStor.GetAccountActionPlans(acntID, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetAccountActionPlans(acntID, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(aAPs, rcv) { t.Errorf("Expecting: %v, received: %v", aAPs, rcv) } - if err := onStor.SetAccountActionPlans(acntID, aAPs2, false); err != nil { + if err := onStor.DataDB().SetAccountActionPlans(acntID, aAPs2, false); err != nil { t.Error(err) } - if rcv, err := onStor.GetAccountActionPlans(acntID, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetAccountActionPlans(acntID, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(expect, rcv) { t.Errorf("Expecting: %v, received: %v", expect, rcv) } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetAccountActionPlans(acntID, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetAccountActionPlans(acntID, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } // - if rcv, err := onStor.GetAccountActionPlans(acntID, false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetAccountActionPlans(acntID, false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(expect, rcv) { t.Errorf("Expecting: %v, received: %v", expect, rcv) } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } - if err := onStor.RemAccountActionPlans(acntID, aAPs2); err != nil { + if err := onStor.DataDB().RemAccountActionPlans(acntID, aAPs2); err != nil { t.Error(err) } - if rcv, err := onStor.GetAccountActionPlans(acntID, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetAccountActionPlans(acntID, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(aAPs, rcv) { t.Errorf("Expecting: %v, received: %v", aAPs, rcv) } - if err := onStor.RemAccountActionPlans(acntID, aAPs); err != nil { + if err := onStor.DataDB().RemAccountActionPlans(acntID, aAPs); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetAccountActionPlans(acntID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetAccountActionPlans(acntID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -1564,13 +1564,13 @@ func testOnStorITCRUDAccount(t *testing.T) { ID: utils.ConcatenatedKey("cgrates.org", "account2"), BalanceMap: map[string]Balances{utils.MONETARY: Balances{&Balance{Value: 10, Weight: 10}}}, } - if _, rcvErr := onStor.GetAccount(acc.ID); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetAccount(acc.ID); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetAccount(acc); err != nil { + if err := onStor.DataDB().SetAccount(acc); err != nil { t.Error(err) } - if rcv, err := onStor.GetAccount(acc.ID); err != nil { + if rcv, err := onStor.DataDB().GetAccount(acc.ID); err != nil { t.Error(err) } else if !reflect.DeepEqual(acc.ID, rcv.ID) { t.Errorf("Expecting: %v, received: %v", acc.ID, rcv.ID) @@ -1579,10 +1579,10 @@ func testOnStorITCRUDAccount(t *testing.T) { } else if !reflect.DeepEqual(acc.BalanceMap[utils.MONETARY][0].Weight, rcv.BalanceMap[utils.MONETARY][0].Weight) { t.Errorf("Expecting: %v, received: %v", acc.BalanceMap[utils.MONETARY][0].Weight, rcv.BalanceMap[utils.MONETARY][0].Weight) } - if err := onStor.RemoveAccount(acc.ID); err != nil { + if err := onStor.DataDB().RemoveAccount(acc.ID); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetAccount(acc.ID); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetAccount(acc.ID); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -1597,13 +1597,13 @@ func testOnStorITCRUDCdrStatsQueue(t *testing.T) { EventTime: time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC).Local(), }}, } - if _, rcvErr := onStor.GetCdrStatsQueue(sq.GetId()); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetCdrStatsQueue(sq.GetId()); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetCdrStatsQueue(sq); err != nil { + if err := onStor.DataDB().SetCdrStatsQueue(sq); err != nil { t.Error(err) } - if rcv, err := onStor.GetCdrStatsQueue(sq.GetId()); err != nil { + if rcv, err := onStor.DataDB().GetCdrStatsQueue(sq.GetId()); err != nil { t.Error(err) } else if !reflect.DeepEqual(sq.Cdrs, rcv.Cdrs) { t.Errorf("Expecting: %v, received: %v", sq.Cdrs, rcv.Cdrs) @@ -1611,7 +1611,7 @@ func testOnStorITCRUDCdrStatsQueue(t *testing.T) { } func testOnStorITCRUDSubscribers(t *testing.T) { - if sbs, err := onStor.GetSubscribers(); err != nil { + if sbs, err := onStor.DataDB().GetSubscribers(); err != nil { t.Error(err) } else if len(sbs) != 0 { t.Errorf("Received subscribers: %+v", sbs) @@ -1620,18 +1620,18 @@ func testOnStorITCRUDSubscribers(t *testing.T) { ExpTime: time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC).Local(), Filters: utils.ParseRSRFieldsMustCompile("^*default", utils.INFIELD_SEP)} sbscID := "testOnStorITCRUDSubscribers" - if err := onStor.SetSubscriber(sbscID, sbsc); err != nil { + if err := onStor.DataDB().SetSubscriber(sbscID, sbsc); err != nil { t.Error(err) } - if rcv, err := onStor.GetSubscribers(); err != nil { + if rcv, err := onStor.DataDB().GetSubscribers(); err != nil { t.Error(err) } else if !reflect.DeepEqual(sbsc.ExpTime, rcv[sbscID].ExpTime) { // Test just ExpTime since RSRField is more complex behind t.Errorf("Expecting: %v, received: %v", sbsc, rcv[sbscID]) } - if err := onStor.RemoveSubscriber(sbscID); err != nil { + if err := onStor.DataDB().RemoveSubscriber(sbscID); err != nil { t.Error(err) } - if sbs, err := onStor.GetSubscribers(); err != nil { + if sbs, err := onStor.DataDB().GetSubscribers(); err != nil { t.Error(err) } else if len(sbs) != 0 { t.Errorf("Received subscribers: %+v", sbs) @@ -1645,26 +1645,26 @@ func testOnStorITCRUDUser(t *testing.T) { "t": "v", }, } - if _, rcvErr := onStor.GetUser(usr.GetId()); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetUser(usr.GetId()); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetUser(usr); err != nil { + if err := onStor.DataDB().SetUser(usr); err != nil { t.Error(err) } - if rcv, err := onStor.GetUser(usr.GetId()); err != nil { + if rcv, err := onStor.DataDB().GetUser(usr.GetId()); err != nil { t.Error(err) } else if !reflect.DeepEqual(usr, rcv) { t.Errorf("Expecting: %v, received: %v", usr, rcv) } - if rcv, err := onStor.GetUsers(); err != nil { + if rcv, err := onStor.DataDB().GetUsers(); err != nil { t.Error(err) } else if !reflect.DeepEqual(usr, rcv[0]) { t.Errorf("Expecting: %v, received: %v", usr, rcv[0]) } - if err := onStor.RemoveUser(usr.GetId()); err != nil { + if err := onStor.DataDB().RemoveUser(usr.GetId()); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetUser(usr.GetId()); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetUser(usr.GetId()); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -1700,37 +1700,37 @@ func testOnStorITCRUDAlias(t *testing.T) { }, } - if _, rcvErr := onStor.GetAlias(als.GetId(), true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetAlias(als.GetId(), true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetAlias(als, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetAlias(als, utils.NonTransactional); err != nil { t.Error(err) } - if rcv, err := onStor.GetAlias(als.GetId(), true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetAlias(als.GetId(), true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(als, rcv) { t.Errorf("Expecting: %v, received: %v", als, rcv) } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetAlias(als.GetId(), false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetAlias(als.GetId(), false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } // - if rcv, err := onStor.GetAlias(als.GetId(), false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetAlias(als.GetId(), false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(als, rcv) { t.Errorf("Expecting: %v, received: %v", als, rcv) } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } - if err := onStor.RemoveAlias(als.GetId(), utils.NonTransactional); err != nil { + if err := onStor.DataDB().RemoveAlias(als.GetId(), utils.NonTransactional); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetAlias(als.GetId(), true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetAlias(als.GetId(), true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -1767,31 +1767,31 @@ func testOnStorITCRUDReverseAlias(t *testing.T) { } rvAlsID := strings.Join([]string{als.Values[1].Pairs["Account"]["dan"], "Account", als.Context}, "") exp := strings.Join([]string{als.Direction, ":", als.Tenant, ":", als.Category, ":", als.Account, ":", als.Subject, ":", als.Context, ":", als.Values[1].DestinationId}, "") - if _, rcvErr := onStor.GetReverseAlias(rvAlsID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetReverseAlias(rvAlsID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetReverseAlias(als, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetReverseAlias(als, utils.NonTransactional); err != nil { t.Error(err) } - if rcv, err := onStor.GetReverseAlias(rvAlsID, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetReverseAlias(rvAlsID, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(exp, rcv[0]) { t.Errorf("Expecting: %v, received: %v", exp, rcv[0]) } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetReverseAlias(rvAlsID, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetReverseAlias(rvAlsID, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } // - if rcv, err := onStor.GetReverseAlias(rvAlsID, false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetReverseAlias(rvAlsID, false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(exp, rcv[0]) { t.Errorf("Expecting: %v, received: %v", exp, rcv[0]) } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } } @@ -1812,37 +1812,37 @@ func testOnStorITCRUDResourceProfile(t *testing.T) { Thresholds: []string{"TEST_ACTIONS"}, UsageTTL: time.Duration(1 * time.Millisecond), } - if _, rcvErr := onStor.GetResourceProfile(rL.Tenant, rL.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetResourceProfile(rL.Tenant, rL.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetResourceProfile(rL); err != nil { + if err := onStor.DataDB().SetResourceProfile(rL); err != nil { t.Error(err) } - if rcv, err := onStor.GetResourceProfile(rL.Tenant, rL.ID, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetResourceProfile(rL.Tenant, rL.ID, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(rL, rcv) { t.Errorf("Expecting: %v, received: %v", rL, rcv) } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetResourceLimit(rL.ID, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetResourceLimit(rL.ID, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } // - if rcv, err := onStor.GetResourceProfile(rL.Tenant, rL.ID, false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetResourceProfile(rL.Tenant, rL.ID, false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(rL, rcv) { t.Errorf("Expecting: %v, received: %v", rL, rcv) } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } - if err := onStor.RemoveResourceProfile(rL.Tenant, rL.ID, utils.NonTransactional); err != nil { + if err := onStor.DataDB().RemoveResourceProfile(rL.Tenant, rL.ID, utils.NonTransactional); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetResourceProfile(rL.Tenant, rL.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetResourceProfile(rL.Tenant, rL.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -1860,26 +1860,26 @@ func testOnStorITCRUDResource(t *testing.T) { }, TTLIdx: []string{"RU1"}, } - if _, rcvErr := onStor.GetResource("cgrates.org", "RL1", true, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetResource("cgrates.org", "RL1", true, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetResource(res); err != nil { + if err := onStor.DataDB().SetResource(res); err != nil { t.Error(err) } - if rcv, err := onStor.GetResource("cgrates.org", "RL1", true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetResource("cgrates.org", "RL1", true, utils.NonTransactional); err != nil { t.Error(err) } else if !(reflect.DeepEqual(res, rcv)) { t.Errorf("Expecting: %v, received: %v", res, rcv) } - if rcv, err := onStor.GetResource("cgrates.org", "RL1", false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetResource("cgrates.org", "RL1", false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(res, rcv) { t.Errorf("Expecting: %v, received: %v", res, rcv) } - if err := onStor.RemoveResource(res.Tenant, res.ID, utils.NonTransactional); err != nil { + if err := onStor.DataDB().RemoveResource(res.Tenant, res.ID, utils.NonTransactional); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetResource(res.Tenant, res.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetResource(res.Tenant, res.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -1894,37 +1894,37 @@ func testOnStorITCRUDTiming(t *testing.T) { StartTime: "00:00:00", EndTime: "", } - if _, rcvErr := onStor.GetTiming(tmg.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetTiming(tmg.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetTiming(tmg, utils.NonTransactional); err != nil { + if err := onStor.DataDB().SetTiming(tmg, utils.NonTransactional); err != nil { t.Error(err) } - if rcv, err := onStor.GetTiming(tmg.ID, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetTiming(tmg.ID, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(tmg, rcv) { t.Errorf("Expecting: %v, received: %v", tmg, rcv) } // FixMe - // if err = onStor.SelectDatabase("13"); err != nil { + // if err = onStor.DataDB().SelectDatabase("13"); err != nil { // t.Error(err) // } - // if _, rcvErr := onStor.GetTiming(tmg.ID, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + // if _, rcvErr := onStor.DataDB().GetTiming(tmg.ID, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { // t.Error(rcvErr) // } // - if rcv, err := onStor.GetTiming(tmg.ID, false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetTiming(tmg.ID, false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(tmg, rcv) { t.Errorf("Expecting: %v, received: %v", tmg, rcv) } - // if err = onStor.SelectDatabase(onStorCfg); err != nil { + // if err = onStor.DataDB().SelectDatabase(onStorCfg); err != nil { // t.Error(err) // } - if err := onStor.RemoveTiming(tmg.ID, utils.NonTransactional); err != nil { + if err := onStor.DataDB().RemoveTiming(tmg.ID, utils.NonTransactional); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetTiming(tmg.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetTiming(tmg.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -1932,10 +1932,10 @@ func testOnStorITCRUDTiming(t *testing.T) { func testOnStorITCRUDHistory(t *testing.T) { time := time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC).Local() ist := &utils.LoadInstance{"Load", "RatingLoad", "Account", time} - if err := onStor.AddLoadHistory(ist, 1, utils.NonTransactional); err != nil { + if err := onStor.DataDB().AddLoadHistory(ist, 1, utils.NonTransactional); err != nil { t.Error(err) } - if rcv, err := onStor.GetLoadHistory(1, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetLoadHistory(1, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(ist, rcv[0]) { t.Errorf("Expecting: %v, received: %v", ist, rcv[0]) @@ -1944,20 +1944,20 @@ func testOnStorITCRUDHistory(t *testing.T) { func testOnStorITCRUDStructVersion(t *testing.T) { CurrentVersion := Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} - if _, rcvErr := onStor.GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetVersions(CurrentVersion, false); err != nil { + if err := onStor.DataDB().SetVersions(CurrentVersion, false); err != nil { t.Error(err) } - if rcv, err := onStor.GetVersions(utils.TBLVersions); err != nil { + if rcv, err := onStor.DataDB().GetVersions(utils.TBLVersions); err != nil { t.Error(err) } else if !reflect.DeepEqual(CurrentVersion, rcv) { t.Errorf("Expecting: %v, received: %v", CurrentVersion, rcv) - } else if err = onStor.RemoveVersions(rcv); err != nil { + } else if err = onStor.DataDB().RemoveVersions(rcv); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -1974,19 +1974,19 @@ func testOnStorITCRUDStatQueueProfile(t *testing.T) { Stored: true, Thresholds: []string{}, } - if _, rcvErr := onStor.GetStatQueueProfile(sq.Tenant, sq.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetStatQueueProfile(sq.Tenant, sq.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } if _, ok := cache.Get(utils.StatQueueProfilePrefix + sq.ID); ok != false { t.Error("Should not be in cache") } - if err := onStor.SetStatQueueProfile(sq); err != nil { + if err := onStor.DataDB().SetStatQueueProfile(sq); err != nil { t.Error(err) } if _, ok := cache.Get(utils.StatQueueProfilePrefix + sq.ID); ok != false { t.Error("Should not be in cache") } - if rcv, err := onStor.GetStatQueueProfile(sq.Tenant, sq.ID, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetStatQueueProfile(sq.Tenant, sq.ID, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(sq, rcv) { t.Errorf("Expecting: %v, received: %v", sq, rcv) @@ -1994,13 +1994,13 @@ func testOnStorITCRUDStatQueueProfile(t *testing.T) { if _, ok := cache.Get(utils.StatQueueProfilePrefix + sq.ID); ok != false { t.Error("Should not be in cache") } - if err := onStor.RemStatQueueProfile(sq.Tenant, sq.ID, utils.NonTransactional); err != nil { + if err := onStor.DataDB().RemStatQueueProfile(sq.Tenant, sq.ID, utils.NonTransactional); err != nil { t.Error(err) } if _, ok := cache.Get(utils.StatQueueProfilePrefix + sq.ID); ok != false { t.Error("Should not be in cache") } - if _, rcvErr := onStor.GetStatQueueProfile(sq.Tenant, sq.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetStatQueueProfile(sq.Tenant, sq.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -2016,7 +2016,7 @@ func testOnStorITCRUDStoredStatQueue(t *testing.T) { "cgrates.org:ev3": false, }, } - msrshled, err := asr.Marshal(onStor.Marshaler()) + msrshled, err := asr.Marshal(onStor.DataDB().Marshaler()) if err != nil { t.Error(err) } @@ -2033,18 +2033,18 @@ func testOnStorITCRUDStoredStatQueue(t *testing.T) { utils.MetaASR: msrshled, }, } - if err := onStor.SetStoredStatQueue(sq); err != nil { + if err := onStor.DataDB().SetStoredStatQueue(sq); err != nil { t.Error(err) } - if rcv, err := onStor.GetStoredStatQueue(sq.Tenant, sq.ID); err != nil { + if rcv, err := onStor.DataDB().GetStoredStatQueue(sq.Tenant, sq.ID); err != nil { t.Error(err) } else if !reflect.DeepEqual(sq, rcv) { t.Errorf("Expecting: %v, received: %v", sq, rcv) } - if err := onStor.RemStoredStatQueue(sq.Tenant, sq.ID); err != nil { + if err := onStor.DataDB().RemStoredStatQueue(sq.Tenant, sq.ID); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetStoredStatQueue(sq.Tenant, sq.ID); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetStoredStatQueue(sq.Tenant, sq.ID); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -2062,30 +2062,30 @@ func testOnStorITCRUDThresholdProfile(t *testing.T) { Weight: 1.4, ActionIDs: []string{}, } - if _, rcvErr := onStor.GetThresholdProfile(th.Tenant, th.ID, + if _, rcvErr := onStor.DataDB().GetThresholdProfile(th.Tenant, th.ID, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetThresholdProfile(th); err != nil { + if err := onStor.DataDB().SetThresholdProfile(th); err != nil { t.Error(err) } - if rcv, err := onStor.GetThresholdProfile(th.Tenant, th.ID, true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetThresholdProfile(th.Tenant, th.ID, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(th, rcv) { t.Errorf("Expecting: %v, received: %v", th, rcv) } - if rcv, err := onStor.GetThresholdProfile(th.Tenant, th.ID, false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetThresholdProfile(th.Tenant, th.ID, false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(th, rcv) { t.Errorf("Expecting: %v, received: %v", th, rcv) } - if err := onStor.RemThresholdProfile(th.Tenant, th.ID, utils.NonTransactional); err != nil { + if err := onStor.DataDB().RemThresholdProfile(th.Tenant, th.ID, utils.NonTransactional); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetThresholdProfile(th.Tenant, th.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetThresholdProfile(th.Tenant, th.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if _, rcvErr := onStor.GetThresholdProfile(th.Tenant, th.ID, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetThresholdProfile(th.Tenant, th.ID, false, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -2096,26 +2096,26 @@ func testOnStorITCRUDThreshold(t *testing.T) { ID: "TH1", Snooze: time.Date(2016, 10, 1, 0, 0, 0, 0, time.UTC).Local(), } - if _, rcvErr := onStor.GetThreshold("cgrates.org", "TH1", true, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetThreshold("cgrates.org", "TH1", true, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetThreshold(res); err != nil { + if err := onStor.DataDB().SetThreshold(res); err != nil { t.Error(err) } - if rcv, err := onStor.GetThreshold("cgrates.org", "TH1", true, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetThreshold("cgrates.org", "TH1", true, utils.NonTransactional); err != nil { t.Error(err) } else if !(reflect.DeepEqual(res, rcv)) { t.Errorf("Expecting: %v, received: %v", res, rcv) } - if rcv, err := onStor.GetThreshold("cgrates.org", "TH1", false, utils.NonTransactional); err != nil { + if rcv, err := onStor.DataDB().GetThreshold("cgrates.org", "TH1", false, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual(res, rcv) { t.Errorf("Expecting: %v, received: %v", res, rcv) } - if err := onStor.RemoveThreshold(res.Tenant, res.ID, utils.NonTransactional); err != nil { + if err := onStor.DataDB().RemoveThreshold(res.Tenant, res.ID, utils.NonTransactional); err != nil { t.Error(err) } - if _, rcvErr := onStor.GetThreshold(res.Tenant, res.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.DataDB().GetThreshold(res.Tenant, res.ID, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } } @@ -2139,7 +2139,7 @@ func testOnStorITCRUDFilter(t *testing.T) { if _, rcvErr := onStor.GetFilter("cgrates.org", "Filter1", true, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.SetFilter(fp); err != nil { + if err := onStor.DataDB().SetFilter(fp); err != nil { t.Error(err) } if rcv, err := onStor.GetFilter("cgrates.org", "Filter1", true, utils.NonTransactional); err != nil { @@ -2152,7 +2152,7 @@ func testOnStorITCRUDFilter(t *testing.T) { } else if !reflect.DeepEqual(fp, rcv) { t.Errorf("Expecting: %v, received: %v", fp, rcv) } - if err := onStor.RemoveFilter(fp.Tenant, fp.ID, utils.NonTransactional); err != nil { + if err := onStor.DataDB().RemoveFilter(fp.Tenant, fp.ID, utils.NonTransactional); err != nil { t.Error(err) } if _, rcvErr := onStor.GetFilter("cgrates.org", "Filter1", true, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound { diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 8f12d912f..b1fea3039 100755 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -126,7 +126,7 @@ type DataDB interface { GetThreshold(string, string, bool, string) (*Threshold, error) SetThreshold(*Threshold) error RemoveThreshold(string, string, string) error - GetFilter(string, string, bool, string) (*Filter, error) + GetFilterDrv(string, string) (*Filter, error) SetFilter(*Filter) error RemoveFilter(string, string, string) error // CacheDataFromDB loads data to cache, prefix represents the cache prefix, IDs should be nil if all available data should be loaded diff --git a/engine/storage_map.go b/engine/storage_map.go index 64aed54fd..d4985afc5 100755 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -1669,28 +1669,14 @@ func (ms *MapStorage) RemoveThreshold(tenant, id string, transactionID string) ( return } -func (ms *MapStorage) GetFilter(tenant, id string, skipCache bool, transactionID string) (r *Filter, err error) { +func (ms *MapStorage) GetFilterDrv(tenant, id string) (r *Filter, err error) { ms.mu.RLock() defer ms.mu.RUnlock() - key := utils.FilterPrefix + utils.ConcatenatedKey(tenant, id) - if !skipCache { - if x, ok := cache.Get(key); ok { - if x != nil { - return x.(*Filter), nil - } - return nil, utils.ErrNotFound - } - } - values, ok := ms.dict[key] + values, ok := ms.dict[utils.FilterPrefix+utils.ConcatenatedKey(tenant, id)] if !ok { - cache.Set(key, nil, cacheCommit(transactionID), transactionID) return nil, utils.ErrNotFound } - err = ms.ms.Unmarshal(values, r) - if err != nil { - return nil, err - } - cache.Set(key, r, cacheCommit(transactionID), transactionID) + err = ms.ms.Unmarshal(values, &r) return } diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index c1c2c2709..fd8a39de0 100755 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -546,7 +546,7 @@ func (ms *MongoStorage) CacheDataFromDB(prfx string, ids []string, mustBeCached _, err = ms.GetThreshold(tntID.Tenant, tntID.ID, true, utils.NonTransactional) case utils.FilterPrefix: tntID := utils.NewTenantID(dataID) - _, err = ms.GetFilter(tntID.Tenant, tntID.ID, true, utils.NonTransactional) + _, err = ms.GetFilterDrv(tntID.Tenant, tntID.ID) } if err != nil { return utils.NewCGRError(utils.MONGO, @@ -2253,27 +2253,15 @@ func (ms *MongoStorage) RemoveThreshold(tenant, id string, transactionID string) return nil } -func (ms *MongoStorage) GetFilter(tenant, id string, skipCache bool, transactionID string) (r *Filter, err error) { - key := utils.FilterPrefix + utils.ConcatenatedKey(tenant, id) - if !skipCache { - if x, ok := cache.Get(key); ok { - if x == nil { - return nil, utils.ErrNotFound - } - return x.(*Filter), nil - } - } +func (ms *MongoStorage) GetFilterDrv(tenant, id string) (r *Filter, err error) { session, col := ms.conn(colFlt) defer session.Close() - r = new(Filter) - if err = col.Find(bson.M{"tenant": tenant, "id": id}).One(r); err != nil { + if err = col.Find(bson.M{"tenant": tenant, "id": id}).One(&r); err != nil { if err == mgo.ErrNotFound { err = utils.ErrNotFound - cache.Set(key, nil, cacheCommit(transactionID), transactionID) } return nil, err } - cache.Set(key, r, cacheCommit(transactionID), transactionID) return } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index ca207d3da..68933c1ee 100755 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -1776,20 +1776,11 @@ func (rs *RedisStorage) RemoveThreshold(tenant, id string, transactionID string) return } -func (rs *RedisStorage) GetFilter(tenant, id string, skipCache bool, transactionID string) (r *Filter, err error) { +func (rs *RedisStorage) GetFilterDrv(tenant, id string) (r *Filter, err error) { key := utils.FilterPrefix + utils.ConcatenatedKey(tenant, id) - if !skipCache { - if x, ok := cache.Get(key); ok { - if x == nil { - return nil, utils.ErrNotFound - } - return x.(*Filter), nil - } - } var values []byte if values, err = rs.Cmd("GET", key).Bytes(); err != nil { if err == redis.ErrRespNil { // did not find the destination - cache.Set(key, nil, cacheCommit(transactionID), transactionID) err = utils.ErrNotFound } return @@ -1797,7 +1788,6 @@ func (rs *RedisStorage) GetFilter(tenant, id string, skipCache bool, transaction if err = rs.ms.Unmarshal(values, &r); err != nil { return } - cache.Set(key, r, cacheCommit(transactionID), transactionID) return } diff --git a/engine/tp_reader.go b/engine/tp_reader.go index ab611998f..10323ac5d 100755 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -2247,7 +2247,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Indexing Filters") } for tenant, mpID := range tpr.flProfiles { - stIdxr, err := NewReqFilterIndexer(tpr.dm, utils.ThresholdsIndex+tenant) + stIdxr, err := NewReqFilterIndexer(tpr.dm, utils.FilterIndex+tenant) if err != nil { return err } From ae9db3f8a5c4e775bca8c54e251fe2aea2660a12 Mon Sep 17 00:00:00 2001 From: TeoV Date: Fri, 13 Oct 2017 14:35:12 +0300 Subject: [PATCH 09/10] Add SetFilter in DataManager --- apier/v1/filters.go | 2 +- engine/datamanager.go | 4 ++ engine/onstor_it_test.go | 2 +- engine/storage_interface.go | 2 +- engine/storage_map.go | 2 +- engine/storage_mongo_datadb.go | 2 +- engine/storage_redis.go | 2 +- engine/tp_reader.go | 122 ++++++++++++++++----------------- 8 files changed, 71 insertions(+), 67 deletions(-) diff --git a/apier/v1/filters.go b/apier/v1/filters.go index 96ced8c3c..d93d53774 100644 --- a/apier/v1/filters.go +++ b/apier/v1/filters.go @@ -28,7 +28,7 @@ func (self *ApierV1) SetFilter(attrs *engine.Filter, reply *string) error { if missing := utils.MissingStructFields(attrs, []string{"Tenant", "ID"}); len(missing) != 0 { return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.DataManager.DataDB().SetFilter(attrs); err != nil { + if err := self.DataManager.SetFilter(attrs); err != nil { return utils.APIErrorHandler(err) } *reply = utils.OK diff --git a/engine/datamanager.go b/engine/datamanager.go index 3d28888e8..4d88ffa1c 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -99,3 +99,7 @@ func (dm *DataManager) GetFilter(tenant, id string, skipCache bool, transactionI cache.Set(key, fltr, cacheCommit(transactionID), transactionID) return } + +func (dm *DataManager) SetFilter(fltr *Filter) (err error) { + return dm.DataDB().SetFilterDrv(fltr) +} diff --git a/engine/onstor_it_test.go b/engine/onstor_it_test.go index 09152d539..b15486775 100644 --- a/engine/onstor_it_test.go +++ b/engine/onstor_it_test.go @@ -2139,7 +2139,7 @@ func testOnStorITCRUDFilter(t *testing.T) { if _, rcvErr := onStor.GetFilter("cgrates.org", "Filter1", true, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound { t.Error(rcvErr) } - if err := onStor.DataDB().SetFilter(fp); err != nil { + if err := onStor.SetFilter(fp); err != nil { t.Error(err) } if rcv, err := onStor.GetFilter("cgrates.org", "Filter1", true, utils.NonTransactional); err != nil { diff --git a/engine/storage_interface.go b/engine/storage_interface.go index b1fea3039..f9ca1e1e8 100755 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -127,7 +127,7 @@ type DataDB interface { SetThreshold(*Threshold) error RemoveThreshold(string, string, string) error GetFilterDrv(string, string) (*Filter, error) - SetFilter(*Filter) error + SetFilterDrv(*Filter) error RemoveFilter(string, string, string) error // CacheDataFromDB loads data to cache, prefix represents the cache prefix, IDs should be nil if all available data should be loaded CacheDataFromDB(prefix string, IDs []string, mustBeCached bool) error // ToDo: Move this to dataManager diff --git a/engine/storage_map.go b/engine/storage_map.go index d4985afc5..653f64483 100755 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -1680,7 +1680,7 @@ func (ms *MapStorage) GetFilterDrv(tenant, id string) (r *Filter, err error) { return } -func (ms *MapStorage) SetFilter(r *Filter) (err error) { +func (ms *MapStorage) SetFilterDrv(r *Filter) (err error) { ms.mu.Lock() defer ms.mu.Unlock() result, err := ms.ms.Marshal(r) diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index fd8a39de0..860e05d16 100755 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -2265,7 +2265,7 @@ func (ms *MongoStorage) GetFilterDrv(tenant, id string) (r *Filter, err error) { return } -func (ms *MongoStorage) SetFilter(r *Filter) (err error) { +func (ms *MongoStorage) SetFilterDrv(r *Filter) (err error) { session, col := ms.conn(colFlt) defer session.Close() _, err = col.Upsert(bson.M{"tenant": r.Tenant, "id": r.ID}, r) diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 68933c1ee..c7f9231aa 100755 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -1791,7 +1791,7 @@ func (rs *RedisStorage) GetFilterDrv(tenant, id string) (r *Filter, err error) { return } -func (rs *RedisStorage) SetFilter(r *Filter) (err error) { +func (rs *RedisStorage) SetFilterDrv(r *Filter) (err error) { result, err := rs.ms.Marshal(r) if err != nil { return err diff --git a/engine/tp_reader.go b/engine/tp_reader.go index 10323ac5d..eb669ed94 100755 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -153,10 +153,10 @@ func (tpr *TpReader) LoadDestinationsFiltered(tag string) (bool, error) { for _, tpDst := range tpDests { dst := NewDestinationFromTPDestination(tpDst) // ToDo: Fix transactions at onlineDB level - if err = tpr.dataStorage.SetDestination(dst, transID); err != nil { + if err = tpr.dm.DataDB().SetDestination(dst, transID); err != nil { cache.RollbackTransaction(transID) } - if err = tpr.dataStorage.SetReverseDestination(dst, transID); err != nil { + if err = tpr.dm.DataDB().SetReverseDestination(dst, transID); err != nil { cache.RollbackTransaction(transID) } } @@ -239,7 +239,7 @@ func (tpr *TpReader) LoadDestinationRates() (err error) { _, destinationExists = tpr.destinations[dr.DestinationId] } if !destinationExists && tpr.dataStorage != nil { - if destinationExists, err = tpr.dataStorage.HasData(utils.DESTINATION_PREFIX, dr.DestinationId); err != nil { + if destinationExists, err = tpr.dm.DataDB().HasData(utils.DESTINATION_PREFIX, dr.DestinationId); err != nil { return err } } @@ -308,7 +308,7 @@ func (tpr *TpReader) LoadRatingPlansFiltered(tag string) (bool, error) { } destsExist := len(dms) != 0 if !destsExist && tpr.dataStorage != nil { - if dbExists, err := tpr.dataStorage.HasData(utils.DESTINATION_PREFIX, drate.DestinationId); err != nil { + if dbExists, err := tpr.dm.DataDB().HasData(utils.DESTINATION_PREFIX, drate.DestinationId); err != nil { return false, err } else if dbExists { destsExist = true @@ -319,12 +319,12 @@ func (tpr *TpReader) LoadRatingPlansFiltered(tag string) (bool, error) { return false, fmt.Errorf("could not get destination for tag %v", drate.DestinationId) } for _, destination := range dms { - tpr.dataStorage.SetDestination(destination, utils.NonTransactional) - tpr.dataStorage.SetReverseDestination(destination, utils.NonTransactional) + tpr.dm.DataDB().SetDestination(destination, utils.NonTransactional) + tpr.dm.DataDB().SetReverseDestination(destination, utils.NonTransactional) } } } - if err := tpr.dataStorage.SetRatingPlan(ratingPlan, utils.NonTransactional); err != nil { + if err := tpr.dm.DataDB().SetRatingPlan(ratingPlan, utils.NonTransactional); err != nil { return false, err } } @@ -381,7 +381,7 @@ func (tpr *TpReader) LoadRatingProfilesFiltered(qriedRpf *utils.TPRatingProfile) } _, exists := tpr.ratingPlans[tpRa.RatingPlanId] if !exists && tpr.dataStorage != nil { - if exists, err = tpr.dataStorage.HasData(utils.RATING_PLAN_PREFIX, tpRa.RatingPlanId); err != nil { + if exists, err = tpr.dm.DataDB().HasData(utils.RATING_PLAN_PREFIX, tpRa.RatingPlanId); err != nil { return err } } @@ -396,7 +396,7 @@ func (tpr *TpReader) LoadRatingProfilesFiltered(qriedRpf *utils.TPRatingProfile) CdrStatQueueIds: strings.Split(tpRa.CdrStatQueueIds, utils.INFIELD_SEP), }) } - if err := tpr.dataStorage.SetRatingProfile(resultRatingProfile, utils.NonTransactional); err != nil { + if err := tpr.dm.DataDB().SetRatingProfile(resultRatingProfile, utils.NonTransactional); err != nil { return err } } @@ -421,7 +421,7 @@ func (tpr *TpReader) LoadRatingProfiles() (err error) { } _, exists := tpr.ratingPlans[tpRa.RatingPlanId] if !exists && tpr.dataStorage != nil { // Only query if there is a connection, eg on dry run there is none - if exists, err = tpr.dataStorage.HasData(utils.RATING_PLAN_PREFIX, tpRa.RatingPlanId); err != nil { + if exists, err = tpr.dm.DataDB().HasData(utils.RATING_PLAN_PREFIX, tpRa.RatingPlanId); err != nil { return err } } @@ -465,7 +465,7 @@ func (tpr *TpReader) LoadSharedGroupsFiltered(tag string, save bool) (err error) } if save { for _, sg := range tpr.sharedGroups { - if err := tpr.dataStorage.SetSharedGroup(sg, utils.NonTransactional); err != nil { + if err := tpr.dm.DataDB().SetSharedGroup(sg, utils.NonTransactional); err != nil { return err } } @@ -495,7 +495,7 @@ func (tpr *TpReader) LoadLCRs() (err error) { } } if !found && tpr.dataStorage != nil { - if keys, err := tpr.dataStorage.GetKeysForPrefix(utils.RATING_PROFILE_PREFIX + ratingProfileSearchKey); err != nil { + if keys, err := tpr.dm.DataDB().GetKeysForPrefix(utils.RATING_PROFILE_PREFIX + ratingProfileSearchKey); err != nil { return fmt.Errorf("[LCR] error querying dataDb %s", err.Error()) } else if len(keys) != 0 { found = true @@ -509,7 +509,7 @@ func (tpr *TpReader) LoadLCRs() (err error) { if rule.DestinationId != "" && rule.DestinationId != utils.ANY { _, found := tpr.destinations[rule.DestinationId] if !found && tpr.dataStorage != nil { - if found, err = tpr.dataStorage.HasData(utils.DESTINATION_PREFIX, rule.DestinationId); err != nil { + if found, err = tpr.dm.DataDB().HasData(utils.DESTINATION_PREFIX, rule.DestinationId); err != nil { return fmt.Errorf("[LCR] error querying dataDb %s", err.Error()) } } @@ -675,7 +675,7 @@ func (tpr *TpReader) LoadActionPlans() (err error) { _, exists := tpr.actions[at.ActionsId] if !exists && tpr.dataStorage != nil { - if exists, err = tpr.dataStorage.HasData(utils.ACTION_PREFIX, at.ActionsId); err != nil { + if exists, err = tpr.dm.DataDB().HasData(utils.ACTION_PREFIX, at.ActionsId); err != nil { return fmt.Errorf("[ActionPlans] Error querying actions: %v - %s", at.ActionsId, err.Error()) } } @@ -829,7 +829,7 @@ func (tpr *TpReader) LoadAccountActionsFiltered(qriedAA *utils.TPAccountActions) if accountAction.ActionPlanId != "" { // get old userBalanceIds exitingAccountIds := make(utils.StringMap) - existingActionPlan, err := tpr.dataStorage.GetActionPlan(accountAction.ActionPlanId, true, utils.NonTransactional) + existingActionPlan, err := tpr.dm.DataDB().GetActionPlan(accountAction.ActionPlanId, true, utils.NonTransactional) if err == nil && existingActionPlan != nil { exitingAccountIds = existingActionPlan.AccountIDs } @@ -899,20 +899,20 @@ func (tpr *TpReader) LoadAccountActionsFiltered(qriedAA *utils.TPAccountActions) AccountID: accID, ActionsID: at.ActionsID, } - if err = tpr.dataStorage.PushTask(t); err != nil { + if err = tpr.dm.DataDB().PushTask(t); err != nil { return err } } } } // write action plan - if err = tpr.dataStorage.SetActionPlan(accountAction.ActionPlanId, actionPlan, false, utils.NonTransactional); err != nil { + if err = tpr.dm.DataDB().SetActionPlan(accountAction.ActionPlanId, actionPlan, false, utils.NonTransactional); err != nil { return errors.New(err.Error() + " (SetActionPlan): " + accountAction.ActionPlanId) } - if err = tpr.dataStorage.SetAccountActionPlans(id, []string{accountAction.ActionPlanId}, false); err != nil { + if err = tpr.dm.DataDB().SetAccountActionPlans(id, []string{accountAction.ActionPlanId}, false); err != nil { return err } - if err = tpr.dataStorage.CacheDataFromDB(utils.AccountActionPlansPrefix, []string{id}, true); err != nil { + if err = tpr.dm.DataDB().CacheDataFromDB(utils.AccountActionPlansPrefix, []string{id}, true); err != nil { return err } } @@ -1012,7 +1012,7 @@ func (tpr *TpReader) LoadAccountActionsFiltered(qriedAA *utils.TPAccountActions) actionIDs = append(actionIDs, atr.ActionsID) } // write action triggers - err = tpr.dataStorage.SetActionTriggers(accountAction.ActionTriggersId, actionTriggers, utils.NonTransactional) + err = tpr.dm.DataDB().SetActionTriggers(accountAction.ActionTriggersId, actionTriggers, utils.NonTransactional) if err != nil { return errors.New(err.Error() + " (SetActionTriggers): " + accountAction.ActionTriggersId) } @@ -1124,12 +1124,12 @@ func (tpr *TpReader) LoadAccountActionsFiltered(qriedAA *utils.TPAccountActions) } // write actions for k, as := range facts { - err = tpr.dataStorage.SetActions(k, as, utils.NonTransactional) + err = tpr.dm.DataDB().SetActions(k, as, utils.NonTransactional) if err != nil { return err } } - ub, err := tpr.dataStorage.GetAccount(id) + ub, err := tpr.dm.DataDB().GetAccount(id) if err != nil { ub = &Account{ ID: id, @@ -1138,7 +1138,7 @@ func (tpr *TpReader) LoadAccountActionsFiltered(qriedAA *utils.TPAccountActions) ub.ActionTriggers = actionTriggers // init counters ub.InitCounters() - if err := tpr.dataStorage.SetAccount(ub); err != nil { + if err := tpr.dm.DataDB().SetAccount(ub); err != nil { return err } } @@ -1224,7 +1224,7 @@ func (tpr *TpReader) LoadDerivedChargersFiltered(filter *utils.TPDerivedChargers } if save { for dcsKey, dcs := range tpr.derivedChargers { - if err := tpr.dataStorage.SetDerivedChargers(dcsKey, dcs, utils.NonTransactional); err != nil { + if err := tpr.dm.DataDB().SetDerivedChargers(dcsKey, dcs, utils.NonTransactional); err != nil { return err } } @@ -1352,7 +1352,7 @@ func (tpr *TpReader) LoadCdrStatsFiltered(tag string, save bool) (err error) { return fmt.Errorf("could not get action triggers for cdr stats id %s: %s", cs.Id, triggerTag) } // write action triggers - err = tpr.dataStorage.SetActionTriggers(triggerTag, triggers, utils.NonTransactional) + err = tpr.dm.DataDB().SetActionTriggers(triggerTag, triggers, utils.NonTransactional) if err != nil { return errors.New(err.Error() + " (SetActionTriggers): " + triggerTag) } @@ -1453,13 +1453,13 @@ func (tpr *TpReader) LoadCdrStatsFiltered(tag string, save bool) (err error) { if save { // write actions for k, as := range tpr.actions { - err = tpr.dataStorage.SetActions(k, as, utils.NonTransactional) + err = tpr.dm.DataDB().SetActions(k, as, utils.NonTransactional) if err != nil { return err } } for _, stat := range tpr.cdrStats { - if err := tpr.dataStorage.SetCdrStats(stat); err != nil { + if err := tpr.dm.DataDB().SetCdrStats(stat); err != nil { return err } } @@ -1485,7 +1485,7 @@ func (tpr *TpReader) LoadUsersFiltered(filter *utils.TPUsers) (bool, error) { for _, up := range tpUser.Profile { user.Profile[up.AttrName] = up.AttrValue } - tpr.dataStorage.SetUser(user) + tpr.dm.DataDB().SetUser(user) } return len(tpUsers) > 0, err } @@ -1546,8 +1546,8 @@ func (tpr *TpReader) LoadAliasesFiltered(filter *utils.TPAliases) (bool, error) } } - tpr.dataStorage.SetAlias(alias, utils.NonTransactional) - tpr.dataStorage.SetReverseAlias(alias, utils.NonTransactional) + tpr.dm.DataDB().SetAlias(alias, utils.NonTransactional) + tpr.dm.DataDB().SetReverseAlias(alias, utils.NonTransactional) return len(tpAliases) > 0, err } @@ -1615,7 +1615,7 @@ func (tpr *TpReader) LoadResourceProfilesFiltered(tag string) error { for tenant, mpID := range mapRsPfls { for id := range mpID { rTid := &utils.TenantID{tenant, id} - if has, err := tpr.dataStorage.HasData(utils.ResourcesPrefix, rTid.TenantID()); err != nil { + if has, err := tpr.dm.DataDB().HasData(utils.ResourcesPrefix, rTid.TenantID()); err != nil { return err } else if !has { tpr.resources = append(tpr.resources, rTid) @@ -1645,7 +1645,7 @@ func (tpr *TpReader) LoadStatsFiltered(tag string) error { for tenant, mpID := range mapSTs { for sqID := range mpID { sqTntID := &utils.TenantID{tenant, sqID} - if has, err := tpr.dataStorage.HasData(utils.StatQueuePrefix, sqTntID.TenantID()); err != nil { + if has, err := tpr.dm.DataDB().HasData(utils.StatQueuePrefix, sqTntID.TenantID()); err != nil { return err } else if !has { tpr.statQueues = append(tpr.statQueues, sqTntID) @@ -1675,7 +1675,7 @@ func (tpr *TpReader) LoadThresholdsFiltered(tag string) error { for tenant, mpID := range mapTHs { for thID := range mpID { thTntID := &utils.TenantID{Tenant: tenant, ID: thID} - if has, err := tpr.dataStorage.HasData(utils.ThresholdPrefix, thTntID.TenantID()); err != nil { + if has, err := tpr.dm.DataDB().HasData(utils.ThresholdPrefix, thTntID.TenantID()); err != nil { return err } else if !has { tpr.thresholds = append(tpr.thresholds, thTntID) @@ -1705,7 +1705,7 @@ func (tpr *TpReader) LoadFilterFiltered(tag string) error { for tenant, mpID := range mapTHs { for thID := range mpID { thTntID := &utils.TenantID{Tenant: tenant, ID: thID} - if has, err := tpr.dataStorage.HasData(utils.FilterPrefix, thTntID.TenantID()); err != nil { + if has, err := tpr.dm.DataDB().HasData(utils.FilterPrefix, thTntID.TenantID()); err != nil { return err } else if !has { tpr.filters = append(tpr.filters, thTntID) @@ -1807,13 +1807,13 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err return errors.New("no database connection") } if flush { // ToDo - //tpr.dataStorage.Flush("") + //tpr.dm.DataDB().Flush("") } if verbose { log.Print("Destinations:") } for _, d := range tpr.destinations { - err = tpr.dataStorage.SetDestination(d, utils.NonTransactional) + err = tpr.dm.DataDB().SetDestination(d, utils.NonTransactional) if err != nil { return err } @@ -1831,7 +1831,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Rating Plans:") } for _, rp := range tpr.ratingPlans { - err = tpr.dataStorage.SetRatingPlan(rp, utils.NonTransactional) + err = tpr.dm.DataDB().SetRatingPlan(rp, utils.NonTransactional) if err != nil { return err } @@ -1843,7 +1843,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Rating Profiles:") } for _, rp := range tpr.ratingProfiles { - err = tpr.dataStorage.SetRatingProfile(rp, utils.NonTransactional) + err = tpr.dm.DataDB().SetRatingProfile(rp, utils.NonTransactional) if err != nil { return err } @@ -1866,7 +1866,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err if verbose { log.Println("\tTask: ", t) } - if err = tpr.dataStorage.PushTask(t); err != nil { + if err = tpr.dm.DataDB().PushTask(t); err != nil { return err } } @@ -1878,13 +1878,13 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err if verbose { log.Println("\tTask: ", t) } - if err = tpr.dataStorage.PushTask(t); err != nil { + if err = tpr.dm.DataDB().PushTask(t); err != nil { return err } } } } - err = tpr.dataStorage.SetActionPlan(k, ap, false, utils.NonTransactional) + err = tpr.dm.DataDB().SetActionPlan(k, ap, false, utils.NonTransactional) if err != nil { return err } @@ -1902,7 +1902,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Action Triggers:") } for k, atrs := range tpr.actionsTriggers { - err = tpr.dataStorage.SetActionTriggers(k, atrs, utils.NonTransactional) + err = tpr.dm.DataDB().SetActionTriggers(k, atrs, utils.NonTransactional) if err != nil { return err } @@ -1914,7 +1914,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Shared Groups:") } for k, sg := range tpr.sharedGroups { - err = tpr.dataStorage.SetSharedGroup(sg, utils.NonTransactional) + err = tpr.dm.DataDB().SetSharedGroup(sg, utils.NonTransactional) if err != nil { return err } @@ -1926,7 +1926,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("LCR Rules:") } for k, lcr := range tpr.lcrs { - err = tpr.dataStorage.SetLCR(lcr, utils.NonTransactional) + err = tpr.dm.DataDB().SetLCR(lcr, utils.NonTransactional) if err != nil { return err } @@ -1938,7 +1938,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Actions:") } for k, as := range tpr.actions { - err = tpr.dataStorage.SetActions(k, as, utils.NonTransactional) + err = tpr.dm.DataDB().SetActions(k, as, utils.NonTransactional) if err != nil { return err } @@ -1950,7 +1950,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Account Actions:") } for _, ub := range tpr.accountActions { - err = tpr.dataStorage.SetAccount(ub) + err = tpr.dm.DataDB().SetAccount(ub) if err != nil { return err } @@ -1962,7 +1962,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Derived Chargers:") } for key, dcs := range tpr.derivedChargers { - err = tpr.dataStorage.SetDerivedChargers(key, dcs, utils.NonTransactional) + err = tpr.dm.DataDB().SetDerivedChargers(key, dcs, utils.NonTransactional) if err != nil { return err } @@ -1974,7 +1974,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("CDR Stats Queues:") } for _, sq := range tpr.cdrStats { - err = tpr.dataStorage.SetCdrStats(sq) + err = tpr.dm.DataDB().SetCdrStats(sq) if err != nil { return err } @@ -1986,7 +1986,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Users:") } for _, u := range tpr.users { - err = tpr.dataStorage.SetUser(u) + err = tpr.dm.DataDB().SetUser(u) if err != nil { return err } @@ -1998,7 +1998,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Aliases:") } for _, al := range tpr.aliases { - err = tpr.dataStorage.SetAlias(al, utils.NonTransactional) + err = tpr.dm.DataDB().SetAlias(al, utils.NonTransactional) if err != nil { return err } @@ -2021,7 +2021,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err if err != nil { return err } - if err = tpr.dataStorage.SetResourceProfile(rsp); err != nil { + if err = tpr.dm.DataDB().SetResourceProfile(rsp); err != nil { return err } if verbose { @@ -2033,7 +2033,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Resources:") } for _, rTid := range tpr.resources { - if err = tpr.dataStorage.SetResource(&Resource{Tenant: rTid.Tenant, ID: rTid.ID, Usages: make(map[string]*ResourceUsage)}); err != nil { + if err = tpr.dm.DataDB().SetResource(&Resource{Tenant: rTid.Tenant, ID: rTid.ID, Usages: make(map[string]*ResourceUsage)}); err != nil { return } if verbose { @@ -2049,7 +2049,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err if err != nil { return err } - if err = tpr.dataStorage.SetStatQueueProfile(st); err != nil { + if err = tpr.dm.DataDB().SetStatQueueProfile(st); err != nil { return err } if verbose { @@ -2086,7 +2086,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err if err != nil { return err } - if err = tpr.dataStorage.SetThresholdProfile(th); err != nil { + if err = tpr.dm.DataDB().SetThresholdProfile(th); err != nil { return err } if verbose { @@ -2098,7 +2098,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Thresholds:") } for _, thd := range tpr.thresholds { - if err = tpr.dataStorage.SetThreshold(&Threshold{Tenant: thd.Tenant, ID: thd.ID}); err != nil { + if err = tpr.dm.DataDB().SetThreshold(&Threshold{Tenant: thd.Tenant, ID: thd.ID}); err != nil { return err } if verbose { @@ -2114,7 +2114,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err if err != nil { return err } - if err = tpr.dataStorage.SetFilter(th); err != nil { + if err = tpr.dm.SetFilter(th); err != nil { return err } if verbose { @@ -2126,7 +2126,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Filters:") } for _, thd := range tpr.filters { - if err = tpr.dataStorage.SetFilter(&Filter{Tenant: thd.Tenant, ID: thd.ID}); err != nil { + if err = tpr.dm.SetFilter(&Filter{Tenant: thd.Tenant, ID: thd.ID}); err != nil { return err } if verbose { @@ -2137,7 +2137,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err log.Print("Timings:") } for _, t := range tpr.timings { - if err = tpr.dataStorage.SetTiming(t, utils.NonTransactional); err != nil { + if err = tpr.dm.DataDB().SetTiming(t, utils.NonTransactional); err != nil { return err } if verbose { @@ -2149,7 +2149,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err if verbose { log.Print("Rebuilding reverse destinations") } - if err = tpr.dataStorage.RebuildReverseForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil { + if err = tpr.dm.DataDB().RebuildReverseForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil { return err } } @@ -2157,7 +2157,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err if verbose { log.Print("Rebuilding account action plans") } - if err = tpr.dataStorage.RebuildReverseForPrefix(utils.AccountActionPlansPrefix); err != nil { + if err = tpr.dm.DataDB().RebuildReverseForPrefix(utils.AccountActionPlansPrefix); err != nil { return err } } @@ -2165,7 +2165,7 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err if verbose { log.Print("Rebuilding reverse aliases") } - if err = tpr.dataStorage.RebuildReverseForPrefix(utils.REVERSE_ALIASES_PREFIX); err != nil { + if err = tpr.dm.DataDB().RebuildReverseForPrefix(utils.REVERSE_ALIASES_PREFIX); err != nil { return err } } From c37c14d265c6f19a6e6d4d0c38763e8a5b358212 Mon Sep 17 00:00:00 2001 From: TeoV Date: Fri, 13 Oct 2017 15:10:24 +0300 Subject: [PATCH 10/10] Add RemoveFilter in DataManager --- apier/v1/filters.go | 2 +- engine/datamanager.go | 9 +++++++++ engine/onstor_it_test.go | 2 +- engine/storage_interface.go | 2 +- engine/storage_map.go | 4 ++-- engine/storage_mongo_datadb.go | 4 +--- engine/storage_redis.go | 3 +-- 7 files changed, 16 insertions(+), 10 deletions(-) diff --git a/apier/v1/filters.go b/apier/v1/filters.go index d93d53774..e598335e3 100644 --- a/apier/v1/filters.go +++ b/apier/v1/filters.go @@ -56,7 +56,7 @@ func (self *ApierV1) RemFilter(arg utils.TenantID, reply *string) error { if missing := utils.MissingStructFields(&arg, []string{"Tenant", "ID"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.DataManager.DataDB().RemoveFilter(arg.Tenant, arg.ID, utils.NonTransactional); err != nil { + if err := self.DataManager.RemoveFilter(arg.Tenant, arg.ID, utils.NonTransactional); err != nil { if err.Error() != utils.ErrNotFound.Error() { err = utils.NewErrServerError(err) } diff --git a/engine/datamanager.go b/engine/datamanager.go index 4d88ffa1c..adba745e8 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -103,3 +103,12 @@ func (dm *DataManager) GetFilter(tenant, id string, skipCache bool, transactionI func (dm *DataManager) SetFilter(fltr *Filter) (err error) { return dm.DataDB().SetFilterDrv(fltr) } + +func (dm *DataManager) RemoveFilter(tenant, id, transactionID string) (err error) { + if err = dm.DataDB().RemoveFilterDrv(tenant, id); err != nil { + return + } + cache.RemKey(utils.FilterPrefix+utils.ConcatenatedKey(tenant, id), + cacheCommit(transactionID), transactionID) + return +} diff --git a/engine/onstor_it_test.go b/engine/onstor_it_test.go index b15486775..a705959eb 100644 --- a/engine/onstor_it_test.go +++ b/engine/onstor_it_test.go @@ -2152,7 +2152,7 @@ func testOnStorITCRUDFilter(t *testing.T) { } else if !reflect.DeepEqual(fp, rcv) { t.Errorf("Expecting: %v, received: %v", fp, rcv) } - if err := onStor.DataDB().RemoveFilter(fp.Tenant, fp.ID, utils.NonTransactional); err != nil { + if err := onStor.RemoveFilter(fp.Tenant, fp.ID, utils.NonTransactional); err != nil { t.Error(err) } if _, rcvErr := onStor.GetFilter("cgrates.org", "Filter1", true, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound { diff --git a/engine/storage_interface.go b/engine/storage_interface.go index f9ca1e1e8..4357739ec 100755 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -128,7 +128,7 @@ type DataDB interface { RemoveThreshold(string, string, string) error GetFilterDrv(string, string) (*Filter, error) SetFilterDrv(*Filter) error - RemoveFilter(string, string, string) error + RemoveFilterDrv(string, string) error // CacheDataFromDB loads data to cache, prefix represents the cache prefix, IDs should be nil if all available data should be loaded CacheDataFromDB(prefix string, IDs []string, mustBeCached bool) error // ToDo: Move this to dataManager } diff --git a/engine/storage_map.go b/engine/storage_map.go index 653f64483..a82009232 100755 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -1691,12 +1691,12 @@ func (ms *MapStorage) SetFilterDrv(r *Filter) (err error) { return } -func (ms *MapStorage) RemoveFilter(tenant, id string, transactionID string) (err error) { +func (ms *MapStorage) RemoveFilterDrv(tenant, id string) (err error) { ms.mu.Lock() defer ms.mu.Unlock() key := utils.FilterPrefix + utils.ConcatenatedKey(tenant, id) delete(ms.dict, key) - cache.RemKey(key, cacheCommit(transactionID), transactionID) + return } diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 860e05d16..0b91975e5 100755 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -2272,13 +2272,11 @@ func (ms *MongoStorage) SetFilterDrv(r *Filter) (err error) { return } -func (ms *MongoStorage) RemoveFilter(tenant, id string, transactionID string) (err error) { +func (ms *MongoStorage) RemoveFilterDrv(tenant, id string) (err error) { session, col := ms.conn(colFlt) defer session.Close() if err = col.Remove(bson.M{"tenant": tenant, "id": id}); err != nil { return } - cache.RemKey(utils.FilterPrefix+utils.ConcatenatedKey(tenant, id), - cacheCommit(transactionID), transactionID) return nil } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index c7f9231aa..dd7f4c934 100755 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -1799,12 +1799,11 @@ func (rs *RedisStorage) SetFilterDrv(r *Filter) (err error) { return rs.Cmd("SET", utils.FilterPrefix+utils.ConcatenatedKey(r.Tenant, r.ID), result).Err } -func (rs *RedisStorage) RemoveFilter(tenant, id string, transactionID string) (err error) { +func (rs *RedisStorage) RemoveFilterDrv(tenant, id string) (err error) { key := utils.FilterPrefix + utils.ConcatenatedKey(tenant, id) if err = rs.Cmd("DEL", key).Err; err != nil { return } - cache.RemKey(key, cacheCommit(transactionID), transactionID) return }