From 95565e8ff088fb8ccacdb735f4c127c82894d822 Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 15 Sep 2017 11:55:50 +0200 Subject: [PATCH] Loader to consider tenant when building up resource indexes --- data/tariffplans/tutorial/Resources.csv | 14 ++-- engine/loader_csv_test.go | 78 ++++++++++---------- engine/resources.go | 5 ++ engine/tp_reader.go | 98 ++++++++++++++----------- utils/coreutils.go | 20 +++++ 5 files changed, 126 insertions(+), 89 deletions(-) diff --git a/data/tariffplans/tutorial/Resources.csv b/data/tariffplans/tutorial/Resources.csv index 30f655dcc..e67f01a71 100755 --- a/data/tariffplans/tutorial/Resources.csv +++ b/data/tariffplans/tutorial/Resources.csv @@ -1,8 +1,8 @@ #Tenant[0],Id[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5],TTL[6],Limit[7],AllocationMessage[8],Blocker[9],Stored[10],Weight[11],Thresholds[12] -Tester,ResGroup1,*string,Account,1001;1002,2014-07-29T15:00:00Z,1s,7,,true,true,20, -Tester,ResGroup1,*string_prefix,Destination,10;20,,,,,,,, -Tester,ResGroup1,*rsr_fields,,Subject(~^1.*1$);Destination(1002),,,,,,,, -Tester,ResGroup2,*destinations,Destination,DST_FS,2014-07-29T15:00:00Z,3600s,8,SPECIAL_1002,true,true,10, -Tester,ResGroup3,*string,Account,3001,2014-07-29T15:00:00Z,1s,3,,true,true,20, -#ResGroup3,*timings,SetupTime,PEAK,,,,,,,, -#ResGroup3,*cdr_stats,,CDRST1:*min_ASR:34;CDRST_1001:*min_ASR:20,,,,,,,, +cgrates.org,ResGroup1,*string,Account,1001;1002,2014-07-29T15:00:00Z,1s,7,,true,true,20, +cgrates.org,ResGroup1,*string_prefix,Destination,10;20,,,,,,,, +cgrates.org,ResGroup1,*rsr_fields,,Subject(~^1.*1$);Destination(1002),,,,,,,, +cgrates.org,ResGroup2,*destinations,Destination,DST_FS,2014-07-29T15:00:00Z,3600s,8,SPECIAL_1002,true,true,10, +cgrates.org,ResGroup3,*string,Account,3001,2014-07-29T15:00:00Z,1s,3,,true,true,20, +#cgrates.org,ResGroup3,*timings,SetupTime,PEAK,,,,,,,, +#cgrates.org,ResGroup3,*cdr_stats,,CDRST1:*min_ASR:34;CDRST_1001:*min_ASR:20,,,,,,,, diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 0ff3b6eee..39c0c3823 100755 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -267,10 +267,10 @@ cgrates.org,mas,true,another,value,10 ` resProfiles = ` #Tenant[0],Id[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5],TTL[6],Limit[7],AllocationMessage[8],Blocker[9],Stored[10],Weight[11],Thresholds[12] -Tester,ResGroup21,*string,HdrAccount,1001;1002,2014-07-29T15:00:00Z,1s,2,call,true,true,10, -Tester,ResGroup21,*string_prefix,HdrDestination,10;20,,,,,,,, -Tester,ResGroup21,*rsr_fields,,HdrSubject(~^1.*1$);HdrDestination(1002),,,,,,,, -Tester,ResGroup22,*destinations,HdrDestination,DST_FS,2014-07-29T15:00:00Z,3600s,2,premium_call,true,true,10, +cgrates.org,ResGroup21,*string,HdrAccount,1001;1002,2014-07-29T15:00:00Z,1s,2,call,true,true,10, +cgrates.org,ResGroup21,*string_prefix,HdrDestination,10;20,,,,,,,, +cgrates.org,ResGroup21,*rsr_fields,,HdrSubject(~^1.*1$);HdrDestination(1002),,,,,,,, +cgrates.org,ResGroup22,*destinations,HdrDestination,DST_FS,2014-07-29T15:00:00Z,3600s,2,premium_call,true,true,10, ` stats = ` #Tenant[0],Id[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5],QueueLength[6],TTL[7],Metrics[8],Blocker[9],Stored[10],Weight[11],Thresholds[12] @@ -1390,44 +1390,46 @@ func TestLoadReverseAliases(t *testing.T) { } func TestLoadResourceProfiles(t *testing.T) { - eResProfiles := map[string]*utils.TPResource{ - "ResGroup21": &utils.TPResource{ - TPid: testTPID, - Tenant: "Tester", - ID: "ResGroup21", - Filters: []*utils.TPRequestFilter{ - &utils.TPRequestFilter{Type: MetaString, FieldName: "HdrAccount", Values: []string{"1001", "1002"}}, - &utils.TPRequestFilter{Type: MetaStringPrefix, FieldName: "HdrDestination", Values: []string{"10", "20"}}, - &utils.TPRequestFilter{Type: MetaRSRFields, Values: []string{"HdrSubject(~^1.*1$)", "HdrDestination(1002)"}}, + eResProfiles := map[string]map[string]*utils.TPResource{ + "cgrates.org": map[string]*utils.TPResource{ + "ResGroup21": &utils.TPResource{ + TPid: testTPID, + Tenant: "cgrates.org", + ID: "ResGroup21", + Filters: []*utils.TPRequestFilter{ + &utils.TPRequestFilter{Type: MetaString, FieldName: "HdrAccount", Values: []string{"1001", "1002"}}, + &utils.TPRequestFilter{Type: MetaStringPrefix, FieldName: "HdrDestination", Values: []string{"10", "20"}}, + &utils.TPRequestFilter{Type: MetaRSRFields, Values: []string{"HdrSubject(~^1.*1$)", "HdrDestination(1002)"}}, + }, + ActivationInterval: &utils.TPActivationInterval{ + ActivationTime: "2014-07-29T15:00:00Z", + }, + UsageTTL: "1s", + AllocationMessage: "call", + Weight: 10, + Limit: "2", }, - ActivationInterval: &utils.TPActivationInterval{ - ActivationTime: "2014-07-29T15:00:00Z", + "ResGroup22": &utils.TPResource{ + TPid: testTPID, + Tenant: "cgrates.org", + ID: "ResGroup22", + Filters: []*utils.TPRequestFilter{ + &utils.TPRequestFilter{Type: MetaDestinations, FieldName: "HdrDestination", Values: []string{"DST_FS"}}, + }, + ActivationInterval: &utils.TPActivationInterval{ + ActivationTime: "2014-07-29T15:00:00Z", + }, + UsageTTL: "3600s", + AllocationMessage: "premium_call", + Blocker: true, + Stored: true, + Weight: 10, + Limit: "2", }, - UsageTTL: "1s", - AllocationMessage: "call", - Weight: 10, - Limit: "2", - }, - "ResGroup22": &utils.TPResource{ - TPid: testTPID, - Tenant: "Tester", - ID: "ResGroup22", - Filters: []*utils.TPRequestFilter{ - &utils.TPRequestFilter{Type: MetaDestinations, FieldName: "HdrDestination", Values: []string{"DST_FS"}}, - }, - ActivationInterval: &utils.TPActivationInterval{ - ActivationTime: "2014-07-29T15:00:00Z", - }, - UsageTTL: "3600s", - AllocationMessage: "premium_call", - Blocker: true, - Stored: true, - Weight: 10, - Limit: "2", }, } - if len(csvr.resProfiles) != len(eResProfiles) { - t.Error("Failed to load resourceProfiles: ", len(csvr.resProfiles)) + if len(csvr.resProfiles["cgrates.org"]) != len(eResProfiles["cgrates.org"]) { + t.Errorf("Failed to load resourceProfiles: %s", utils.ToIJSON(csvr.resProfiles)) } else if !reflect.DeepEqual(eResProfiles["ResGroup22"], csvr.resProfiles["ResGroup22"]) { t.Errorf("Expecting: %+v, received: %+v", eResProfiles["ResGroup22"], csvr.resProfiles["ResGroup22"]) diff --git a/engine/resources.go b/engine/resources.go index 40a2c5ba7..5936ba07e 100755 --- a/engine/resources.go +++ b/engine/resources.go @@ -52,6 +52,11 @@ type ResourceProfile struct { Thresholds []string // Thresholds to check after changing Limit } +// TenantID returns unique identifier of the ResourceProfile in a multi-tenant environment +func (rp *ResourceProfile) TenantID() string { + return utils.ConcatenatedKey(rp.Tenant, rp.ID) +} + // ResourceUsage represents an usage counted type ResourceUsage struct { Tenant string diff --git a/engine/tp_reader.go b/engine/tp_reader.go index d1959c0de..62d509641 100755 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -53,11 +53,11 @@ type TpReader struct { cdrStats map[string]*CdrStats users map[string]*UserProfile aliases map[string]*Alias - resProfiles map[string]*utils.TPResource + resProfiles map[string]map[string]*utils.TPResource sqProfiles map[string]*utils.TPStats thresholds map[string]*utils.TPThreshold - resources []string // IDs of resources which need creation based on resourceProfiles - statQueues []string // IDs of statQueues which need creation based on statQueueProfiles + resources []*utils.TenantID // IDs of resources which need creation based on resourceProfiles + statQueues []string // IDs of statQueues which need creation based on statQueueProfiles revDests, revAliases, @@ -129,7 +129,7 @@ func (tpr *TpReader) Init() { tpr.users = make(map[string]*UserProfile) tpr.aliases = make(map[string]*Alias) tpr.derivedChargers = make(map[string]*utils.DerivedChargers) - tpr.resProfiles = make(map[string]*utils.TPResource) + tpr.resProfiles = make(map[string]map[string]*utils.TPResource) tpr.sqProfiles = make(map[string]*utils.TPStats) tpr.thresholds = make(map[string]*utils.TPThreshold) tpr.revDests = make(map[string][]string) @@ -1599,16 +1599,22 @@ func (tpr *TpReader) LoadResourceProfilesFiltered(tag string) error { if err != nil { return err } - mapRsPs := make(map[string]*utils.TPResource) + mapRsPfls := make(map[string]map[string]*utils.TPResource) for _, rl := range rls { - mapRsPs[rl.ID] = rl + if _, has := mapRsPfls[rl.Tenant]; !has { + mapRsPfls[rl.Tenant] = make(map[string]*utils.TPResource) + } + mapRsPfls[rl.Tenant][rl.ID] = rl } - tpr.resProfiles = mapRsPs - for rID := range mapRsPs { - if has, err := tpr.dataStorage.HasData(utils.ResourcesPrefix, rID); err != nil { - return err - } else if !has { - tpr.resources = append(tpr.resources, rID) + tpr.resProfiles = mapRsPfls + for tenant, mpID := range mapRsPfls { + for id := range mpID { + rTid := &utils.TenantID{tenant, id} + if has, err := tpr.dataStorage.HasData(utils.ResourcesPrefix, rTid.TenantID()); err != nil { + return err + } else if !has { + tpr.resources = append(tpr.resources, rTid) + } } } return nil @@ -1952,27 +1958,29 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err if verbose { log.Print("ResourceProfiles:") } - for _, tpRsp := range tpr.resProfiles { - rsp, err := APItoResource(tpRsp, tpr.timezone) - if err != nil { - return err - } - if err = tpr.dataStorage.SetResourceProfile(rsp, utils.NonTransactional); err != nil { - return err - } - if verbose { - log.Print("\t", rsp.ID) + for _, mpID := range tpr.resProfiles { + for _, tpRsp := range mpID { + rsp, err := APItoResource(tpRsp, tpr.timezone) + if err != nil { + return err + } + if err = tpr.dataStorage.SetResourceProfile(rsp, utils.NonTransactional); err != nil { + return err + } + if verbose { + log.Print("\t", rsp.TenantID()) + } } } if verbose { log.Print("Resources:") } - for _, rID := range tpr.resources { - if err = tpr.dataStorage.SetResource(&Resource{ID: rID, Usages: make(map[string]*ResourceUsage)}); err != nil { + for _, rTid := range tpr.resources { + if err = tpr.dataStorage.SetResource(&Resource{Tenant: rTid.Tenant, ID: rTid.ID, Usages: make(map[string]*ResourceUsage)}); err != nil { return } if verbose { - log.Print("\t", rID) + log.Print("\t", rTid.TenantID()) } } if verbose { @@ -2057,22 +2065,24 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err if verbose { log.Print("Indexing resource profiles") } - rlIdxr, err := NewReqFilterIndexer(tpr.dataStorage, utils.ResourceProfilesIndex) - if err != nil { - return err - } - for _, tpRL := range tpr.resProfiles { - if rl, err := APItoResource(tpRL, tpr.timezone); err != nil { + for tenant, mpID := range tpr.resProfiles { + rlIdxr, err := NewReqFilterIndexer(tpr.dataStorage, utils.ResourceProfilesIndex+tenant) + if err != nil { + return err + } + for _, tpRL := range mpID { + if rl, err := APItoResource(tpRL, tpr.timezone); err != nil { + return err + } else { + rlIdxr.IndexFilters(rl.ID, rl.Filters) + } + } + if verbose { + log.Printf("Indexed ResourceProfile tenant: %s keys: %+v", tenant, rlIdxr.ChangedKeys().Slice()) + } + if err := rlIdxr.StoreIndexes(); err != nil { return err - } else { - rlIdxr.IndexFilters(rl.ID, rl.Filters) } - } - if verbose { - log.Printf("Indexed ResourceProfile keys: %+v", rlIdxr.ChangedKeys().Slice()) - } - if err := rlIdxr.StoreIndexes(); err != nil { - return err } } if len(tpr.sqProfiles) > 0 { @@ -2294,11 +2304,11 @@ func (tpr *TpReader) GetLoadedIds(categ string) ([]string, error) { } return keys, nil case utils.ResourceProfilesPrefix: - keys := make([]string, len(tpr.resProfiles)) - i := 0 - for k := range tpr.resProfiles { - keys[i] = k - i++ + keys := make([]string, 0) + for tenant, mpID := range tpr.resProfiles { + for id := range mpID { + keys = append(keys, utils.ConcatenatedKey(tenant, id)) + } } return keys, nil case utils.ACTION_TRIGGER_PREFIX: diff --git a/utils/coreutils.go b/utils/coreutils.go index 7f0240d7f..df8db9ec3 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -759,3 +759,23 @@ func AppendToFile(fName, text string) error { f.Close() return nil } + +func NewTenantID(tntID string) *TenantID { + if strings.Index(tntID, CONCATENATED_KEY_SEP) == -1 { // no :, ID without Tenant + return &TenantID{ID: tntID} + } + tIDSplt := strings.Split(tntID, CONCATENATED_KEY_SEP) + if len(tIDSplt) == 1 { // only Tenant present + return &TenantID{Tenant: tIDSplt[0]} + } + return &TenantID{Tenant: tIDSplt[0], ID: tIDSplt[1]} +} + +type TenantID struct { + Tenant string + ID string +} + +func (tID *TenantID) TenantID() string { + return ConcatenatedKey(tID.Tenant, tID.ID) +}