Loader to consider tenant when building up resource indexes

data/tariffplans/tutorial/Resources.csv

@@ -1,8 +1,8 @@
 #Tenant[0],Id[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5],TTL[6],Limit[7],AllocationMessage[8],Blocker[9],Stored[10],Weight[11],Thresholds[12]
-Tester,ResGroup1,*string,Account,1001;1002,2014-07-29T15:00:00Z,1s,7,,true,true,20,
-Tester,ResGroup1,*string_prefix,Destination,10;20,,,,,,,,
-Tester,ResGroup1,*rsr_fields,,Subject(~^1.*1$);Destination(1002),,,,,,,,
-Tester,ResGroup2,*destinations,Destination,DST_FS,2014-07-29T15:00:00Z,3600s,8,SPECIAL_1002,true,true,10,
-Tester,ResGroup3,*string,Account,3001,2014-07-29T15:00:00Z,1s,3,,true,true,20,
-#ResGroup3,*timings,SetupTime,PEAK,,,,,,,,
-#ResGroup3,*cdr_stats,,CDRST1:*min_ASR:34;CDRST_1001:*min_ASR:20,,,,,,,,
+cgrates.org,ResGroup1,*string,Account,1001;1002,2014-07-29T15:00:00Z,1s,7,,true,true,20,
+cgrates.org,ResGroup1,*string_prefix,Destination,10;20,,,,,,,,
+cgrates.org,ResGroup1,*rsr_fields,,Subject(~^1.*1$);Destination(1002),,,,,,,,
+cgrates.org,ResGroup2,*destinations,Destination,DST_FS,2014-07-29T15:00:00Z,3600s,8,SPECIAL_1002,true,true,10,
+cgrates.org,ResGroup3,*string,Account,3001,2014-07-29T15:00:00Z,1s,3,,true,true,20,
+#cgrates.org,ResGroup3,*timings,SetupTime,PEAK,,,,,,,,
+#cgrates.org,ResGroup3,*cdr_stats,,CDRST1:*min_ASR:34;CDRST_1001:*min_ASR:20,,,,,,,,

engine/loader_csv_test.go

@@ -267,10 +267,10 @@ cgrates.org,mas,true,another,value,10
 `
 	resProfiles = `
 #Tenant[0],Id[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5],TTL[6],Limit[7],AllocationMessage[8],Blocker[9],Stored[10],Weight[11],Thresholds[12]
-Tester,ResGroup21,*string,HdrAccount,1001;1002,2014-07-29T15:00:00Z,1s,2,call,true,true,10,
-Tester,ResGroup21,*string_prefix,HdrDestination,10;20,,,,,,,,
-Tester,ResGroup21,*rsr_fields,,HdrSubject(~^1.*1$);HdrDestination(1002),,,,,,,,
-Tester,ResGroup22,*destinations,HdrDestination,DST_FS,2014-07-29T15:00:00Z,3600s,2,premium_call,true,true,10,
+cgrates.org,ResGroup21,*string,HdrAccount,1001;1002,2014-07-29T15:00:00Z,1s,2,call,true,true,10,
+cgrates.org,ResGroup21,*string_prefix,HdrDestination,10;20,,,,,,,,
+cgrates.org,ResGroup21,*rsr_fields,,HdrSubject(~^1.*1$);HdrDestination(1002),,,,,,,,
+cgrates.org,ResGroup22,*destinations,HdrDestination,DST_FS,2014-07-29T15:00:00Z,3600s,2,premium_call,true,true,10,
 `
 	stats = `
 #Tenant[0],Id[1],FilterType[2],FilterFieldName[3],FilterFieldValues[4],ActivationInterval[5],QueueLength[6],TTL[7],Metrics[8],Blocker[9],Stored[10],Weight[11],Thresholds[12]
@@ -1390,44 +1390,46 @@ func TestLoadReverseAliases(t *testing.T) {
 }
 
 func TestLoadResourceProfiles(t *testing.T) {
-	eResProfiles := map[string]*utils.TPResource{
-		"ResGroup21": &utils.TPResource{
-			TPid:   testTPID,
-			Tenant: "Tester",
-			ID:     "ResGroup21",
-			Filters: []*utils.TPRequestFilter{
-				&utils.TPRequestFilter{Type: MetaString, FieldName: "HdrAccount", Values: []string{"1001", "1002"}},
-				&utils.TPRequestFilter{Type: MetaStringPrefix, FieldName: "HdrDestination", Values: []string{"10", "20"}},
-				&utils.TPRequestFilter{Type: MetaRSRFields, Values: []string{"HdrSubject(~^1.*1$)", "HdrDestination(1002)"}},
+	eResProfiles := map[string]map[string]*utils.TPResource{
+		"cgrates.org": map[string]*utils.TPResource{
+			"ResGroup21": &utils.TPResource{
+				TPid:   testTPID,
+				Tenant: "cgrates.org",
+				ID:     "ResGroup21",
+				Filters: []*utils.TPRequestFilter{
+					&utils.TPRequestFilter{Type: MetaString, FieldName: "HdrAccount", Values: []string{"1001", "1002"}},
+					&utils.TPRequestFilter{Type: MetaStringPrefix, FieldName: "HdrDestination", Values: []string{"10", "20"}},
+					&utils.TPRequestFilter{Type: MetaRSRFields, Values: []string{"HdrSubject(~^1.*1$)", "HdrDestination(1002)"}},
+				},
+				ActivationInterval: &utils.TPActivationInterval{
+					ActivationTime: "2014-07-29T15:00:00Z",
+				},
+				UsageTTL:          "1s",
+				AllocationMessage: "call",
+				Weight:            10,
+				Limit:             "2",
 			},
-			ActivationInterval: &utils.TPActivationInterval{
-				ActivationTime: "2014-07-29T15:00:00Z",
+			"ResGroup22": &utils.TPResource{
+				TPid:   testTPID,
+				Tenant: "cgrates.org",
+				ID:     "ResGroup22",
+				Filters: []*utils.TPRequestFilter{
+					&utils.TPRequestFilter{Type: MetaDestinations, FieldName: "HdrDestination", Values: []string{"DST_FS"}},
+				},
+				ActivationInterval: &utils.TPActivationInterval{
+					ActivationTime: "2014-07-29T15:00:00Z",
+				},
+				UsageTTL:          "3600s",
+				AllocationMessage: "premium_call",
+				Blocker:           true,
+				Stored:            true,
+				Weight:            10,
+				Limit:             "2",
 			},
-			UsageTTL:          "1s",
-			AllocationMessage: "call",
-			Weight:            10,
-			Limit:             "2",
-		},
-		"ResGroup22": &utils.TPResource{
-			TPid:   testTPID,
-			Tenant: "Tester",
-			ID:     "ResGroup22",
-			Filters: []*utils.TPRequestFilter{
-				&utils.TPRequestFilter{Type: MetaDestinations, FieldName: "HdrDestination", Values: []string{"DST_FS"}},
-			},
-			ActivationInterval: &utils.TPActivationInterval{
-				ActivationTime: "2014-07-29T15:00:00Z",
-			},
-			UsageTTL:          "3600s",
-			AllocationMessage: "premium_call",
-			Blocker:           true,
-			Stored:            true,
-			Weight:            10,
-			Limit:             "2",
 		},
 	}
-	if len(csvr.resProfiles) != len(eResProfiles) {
-		t.Error("Failed to load resourceProfiles: ", len(csvr.resProfiles))
+	if len(csvr.resProfiles["cgrates.org"]) != len(eResProfiles["cgrates.org"]) {
+		t.Errorf("Failed to load resourceProfiles: %s", utils.ToIJSON(csvr.resProfiles))
 	} else if !reflect.DeepEqual(eResProfiles["ResGroup22"], csvr.resProfiles["ResGroup22"]) {
 		t.Errorf("Expecting: %+v, received: %+v", eResProfiles["ResGroup22"], csvr.resProfiles["ResGroup22"])
 

engine/resources.go

@@ -52,6 +52,11 @@ type ResourceProfile struct {
 	Thresholds         []string // Thresholds to check after changing Limit
 }
 
+// TenantID returns unique identifier of the ResourceProfile in a multi-tenant environment
+func (rp *ResourceProfile) TenantID() string {
+	return utils.ConcatenatedKey(rp.Tenant, rp.ID)
+}
+
 // ResourceUsage represents an usage counted
 type ResourceUsage struct {
 	Tenant     string

engine/tp_reader.go

@@ -53,11 +53,11 @@ type TpReader struct {
 	cdrStats         map[string]*CdrStats
 	users            map[string]*UserProfile
 	aliases          map[string]*Alias
-	resProfiles      map[string]*utils.TPResource
+	resProfiles      map[string]map[string]*utils.TPResource
 	sqProfiles       map[string]*utils.TPStats
 	thresholds       map[string]*utils.TPThreshold
-	resources        []string // IDs of resources which need creation based on resourceProfiles
-	statQueues       []string // IDs of statQueues which need creation based on statQueueProfiles
+	resources        []*utils.TenantID // IDs of resources which need creation based on resourceProfiles
+	statQueues       []string          // IDs of statQueues which need creation based on statQueueProfiles
 
 	revDests,
 	revAliases,
@@ -129,7 +129,7 @@ func (tpr *TpReader) Init() {
 	tpr.users = make(map[string]*UserProfile)
 	tpr.aliases = make(map[string]*Alias)
 	tpr.derivedChargers = make(map[string]*utils.DerivedChargers)
-	tpr.resProfiles = make(map[string]*utils.TPResource)
+	tpr.resProfiles = make(map[string]map[string]*utils.TPResource)
 	tpr.sqProfiles = make(map[string]*utils.TPStats)
 	tpr.thresholds = make(map[string]*utils.TPThreshold)
 	tpr.revDests = make(map[string][]string)
@@ -1599,16 +1599,22 @@ func (tpr *TpReader) LoadResourceProfilesFiltered(tag string) error {
 	if err != nil {
 		return err
 	}
-	mapRsPs := make(map[string]*utils.TPResource)
+	mapRsPfls := make(map[string]map[string]*utils.TPResource)
 	for _, rl := range rls {
-		mapRsPs[rl.ID] = rl
+		if _, has := mapRsPfls[rl.Tenant]; !has {
+			mapRsPfls[rl.Tenant] = make(map[string]*utils.TPResource)
+		}
+		mapRsPfls[rl.Tenant][rl.ID] = rl
 	}
-	tpr.resProfiles = mapRsPs
-	for rID := range mapRsPs {
-		if has, err := tpr.dataStorage.HasData(utils.ResourcesPrefix, rID); err != nil {
-			return err
-		} else if !has {
-			tpr.resources = append(tpr.resources, rID)
+	tpr.resProfiles = mapRsPfls
+	for tenant, mpID := range mapRsPfls {
+		for id := range mpID {
+			rTid := &utils.TenantID{tenant, id}
+			if has, err := tpr.dataStorage.HasData(utils.ResourcesPrefix, rTid.TenantID()); err != nil {
+				return err
+			} else if !has {
+				tpr.resources = append(tpr.resources, rTid)
+			}
 		}
 	}
 	return nil
@@ -1952,27 +1958,29 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err
 	if verbose {
 		log.Print("ResourceProfiles:")
 	}
-	for _, tpRsp := range tpr.resProfiles {
-		rsp, err := APItoResource(tpRsp, tpr.timezone)
-		if err != nil {
-			return err
-		}
-		if err = tpr.dataStorage.SetResourceProfile(rsp, utils.NonTransactional); err != nil {
-			return err
-		}
-		if verbose {
-			log.Print("\t", rsp.ID)
+	for _, mpID := range tpr.resProfiles {
+		for _, tpRsp := range mpID {
+			rsp, err := APItoResource(tpRsp, tpr.timezone)
+			if err != nil {
+				return err
+			}
+			if err = tpr.dataStorage.SetResourceProfile(rsp, utils.NonTransactional); err != nil {
+				return err
+			}
+			if verbose {
+				log.Print("\t", rsp.TenantID())
+			}
 		}
 	}
 	if verbose {
 		log.Print("Resources:")
 	}
-	for _, rID := range tpr.resources {
-		if err = tpr.dataStorage.SetResource(&Resource{ID: rID, Usages: make(map[string]*ResourceUsage)}); err != nil {
+	for _, rTid := range tpr.resources {
+		if err = tpr.dataStorage.SetResource(&Resource{Tenant: rTid.Tenant, ID: rTid.ID, Usages: make(map[string]*ResourceUsage)}); err != nil {
 			return
 		}
 		if verbose {
-			log.Print("\t", rID)
+			log.Print("\t", rTid.TenantID())
 		}
 	}
 	if verbose {
@@ -2057,22 +2065,24 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose, disable_reverse bool) (err
 			if verbose {
 				log.Print("Indexing resource profiles")
 			}
-			rlIdxr, err := NewReqFilterIndexer(tpr.dataStorage, utils.ResourceProfilesIndex)
-			if err != nil {
-				return err
-			}
-			for _, tpRL := range tpr.resProfiles {
-				if rl, err := APItoResource(tpRL, tpr.timezone); err != nil {
+			for tenant, mpID := range tpr.resProfiles {
+				rlIdxr, err := NewReqFilterIndexer(tpr.dataStorage, utils.ResourceProfilesIndex+tenant)
+				if err != nil {
+					return err
+				}
+				for _, tpRL := range mpID {
+					if rl, err := APItoResource(tpRL, tpr.timezone); err != nil {
+						return err
+					} else {
+						rlIdxr.IndexFilters(rl.ID, rl.Filters)
+					}
+				}
+				if verbose {
+					log.Printf("Indexed ResourceProfile tenant: %s keys: %+v", tenant, rlIdxr.ChangedKeys().Slice())
+				}
+				if err := rlIdxr.StoreIndexes(); err != nil {
 					return err
-				} else {
-					rlIdxr.IndexFilters(rl.ID, rl.Filters)
 				}
-			}
-			if verbose {
-				log.Printf("Indexed ResourceProfile keys: %+v", rlIdxr.ChangedKeys().Slice())
-			}
-			if err := rlIdxr.StoreIndexes(); err != nil {
-				return err
 			}
 		}
 		if len(tpr.sqProfiles) > 0 {
@@ -2294,11 +2304,11 @@ func (tpr *TpReader) GetLoadedIds(categ string) ([]string, error) {
 		}
 		return keys, nil
 	case utils.ResourceProfilesPrefix:
-		keys := make([]string, len(tpr.resProfiles))
-		i := 0
-		for k := range tpr.resProfiles {
-			keys[i] = k
-			i++
+		keys := make([]string, 0)
+		for tenant, mpID := range tpr.resProfiles {
+			for id := range mpID {
+				keys = append(keys, utils.ConcatenatedKey(tenant, id))
+			}
 		}
 		return keys, nil
 	case utils.ACTION_TRIGGER_PREFIX:

utils/coreutils.go

@@ -759,3 +759,23 @@ func AppendToFile(fName, text string) error {
 	f.Close()
 	return nil
 }
+
+func NewTenantID(tntID string) *TenantID {
+	if strings.Index(tntID, CONCATENATED_KEY_SEP) == -1 { // no :, ID without Tenant
+		return &TenantID{ID: tntID}
+	}
+	tIDSplt := strings.Split(tntID, CONCATENATED_KEY_SEP)
+	if len(tIDSplt) == 1 { // only Tenant present
+		return &TenantID{Tenant: tIDSplt[0]}
+	}
+	return &TenantID{Tenant: tIDSplt[0], ID: tIDSplt[1]}
+}
+
+type TenantID struct {
+	Tenant string
+	ID     string
+}
+
+func (tID *TenantID) TenantID() string {
+	return ConcatenatedKey(tID.Tenant, tID.ID)
+}