Merge pull request #717 from TeoV/master

Implement Resource in Cache and delete comments

apier/v1/apier.go

@@ -926,7 +926,20 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 	if err = self.DataDB.CacheDataFromDB(utils.REVERSE_ALIASES_PREFIX, dataIDs, true); err != nil {
 		return
 	}
-	// Resources
+	// ResourceConfig
+	dataIDs = make([]string, 0)
+	if attrs.ResourceConfigIDs == nil {
+		dataIDs = nil // Reload all
+	} else if len(*attrs.ResourceConfigIDs) > 0 {
+		dataIDs = make([]string, len(*attrs.ResourceConfigIDs))
+		for idx, dId := range *attrs.ResourceConfigIDs {
+			dataIDs[idx] = dId
+		}
+	}
+	if err = self.DataDB.CacheDataFromDB(utils.ResourceConfigsPrefix, dataIDs, true); err != nil {
+		return
+	}
+	// Resource
 	dataIDs = make([]string, 0)
 	if attrs.ResourceIDs == nil {
 		dataIDs = nil // Reload all
@@ -936,7 +949,7 @@ func (self *ApierV1) ReloadCache(attrs utils.AttrReloadCache, reply *string) (er
 			dataIDs[idx] = dId
 		}
 	}
-	if err = self.DataDB.CacheDataFromDB(utils.ResourceConfigsPrefix, dataIDs, true); err != nil {
+	if err = self.DataDB.CacheDataFromDB(utils.ResourcesPrefix, dataIDs, true); err != nil {
 		return
 	}
 	*reply = utils.OK
@@ -947,7 +960,7 @@ func (self *ApierV1) LoadCache(args utils.AttrReloadCache, reply *string) (err e
 	if args.FlushAll {
 		cache.Flush()
 	}
-	var dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rlIDs []string
+	var dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rlIDs, resIDs []string
 	if args.DestinationIDs == nil {
 		dstIDs = nil
 	} else {
@@ -1013,15 +1026,21 @@ func (self *ApierV1) LoadCache(args utils.AttrReloadCache, reply *string) (err e
 	} else {
 		rvAlsIDs = *args.ReverseAliasIDs
 	}
-	if args.ResourceIDs == nil {
+	if args.ResourceConfigIDs == nil {
 		rlIDs = nil
 	} else {
-		rlIDs = *args.ResourceIDs
+		rlIDs = *args.ResourceConfigIDs
 	}
+	if args.ResourceIDs == nil {
+		resIDs = nil
+	} else {
+		resIDs = *args.ResourceIDs
+	}
+
 	if err := self.DataDB.LoadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs); err != nil {
 		return utils.NewErrServerError(err)
 	}
-	if err := self.DataDB.LoadAccountingCache(alsIDs, rvAlsIDs, rlIDs); err != nil {
+	if err := self.DataDB.LoadAccountingCache(alsIDs, rvAlsIDs, rlIDs, resIDs); err != nil {
 		return utils.NewErrServerError(err)
 	}
 	*reply = utils.OK

apier/v1/resourcesv1_it_test.go

@@ -19,6 +19,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 package v1
 
+/*
 import (
 	"net/rpc"
 	"net/rpc/jsonrpc"
@@ -361,3 +362,4 @@ func testV1RsStopEngine(t *testing.T) {
 		t.Error(err)
 	}
 }
+*/

cmd/cgr-engine/rater.go

@@ -42,7 +42,7 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC
 	waitTasks = append(waitTasks, cacheTaskChan)
 	go func() {
 		defer close(cacheTaskChan)
-		var dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rlIDs []string
+		var dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs, alsIDs, rvAlsIDs, rlIDs, resIDs []string
 		if cCfg, has := cfg.CacheConfig[utils.CacheDestinations]; !has || !cCfg.Precache {
 			dstIDs = make([]string, 0) // Don't cache any
 		}
@@ -85,13 +85,17 @@ func startRater(internalRaterChan chan rpcclient.RpcClientConnection, cacheDoneC
 		if cCfg, has := cfg.CacheConfig[utils.CacheResourceConfigs]; !has || !cCfg.Precache {
 			rlIDs = make([]string, 0)
 		}
+		if cCfg, has := cfg.CacheConfig[utils.CacheResources]; !has || !cCfg.Precache {
+			resIDs = make([]string, 0)
+		}
+
 		// ToDo: Add here timings
 		if err := dataDB.LoadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs, aplIDs, aapIDs, atrgIDs, sgIDs, lcrIDs, dcIDs); err != nil {
 			utils.Logger.Crit(fmt.Sprintf("<RALs> Cache rating error: %s", err.Error()))
 			exitChan <- true
 			return
 		}
-		if err := dataDB.LoadAccountingCache(alsIDs, rvAlsIDs, rlIDs); err != nil {
+		if err := dataDB.LoadAccountingCache(alsIDs, rvAlsIDs, rlIDs, resIDs); err != nil {
 			utils.Logger.Crit(fmt.Sprintf("<RALs> Cache accounting error: %s", err.Error()))
 			exitChan <- true
 			return

cmd/cgr-tester/cgr-tester.go

@@ -68,7 +68,7 @@ func durInternalRater(cd *engine.CallDescriptor) (time.Duration, error) {
 	if err := dataDb.LoadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil); err != nil {
 		return nilDuration, fmt.Errorf("Cache rating error: %s", err.Error())
 	}
-	if err := dataDb.LoadAccountingCache(nil, nil, nil); err != nil {
+	if err := dataDb.LoadAccountingCache(nil, nil, nil, nil); err != nil {
 		return nilDuration, fmt.Errorf("Cache accounting error: %s", err.Error())
 	}
 	log.Printf("Runnning %d cycles...", *runs)

engine/loader_csv_test.go

@@ -348,7 +348,7 @@ func init() {
 	csvr.WriteToDatabase(false, false, false)
 	cache.Flush()
 	dataStorage.LoadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
-	dataStorage.LoadAccountingCache(nil, nil, nil)
+	dataStorage.LoadAccountingCache(nil, nil, nil, nil)
 }
 
 func TestLoadDestinations(t *testing.T) {

engine/onstor_it_test.go

@@ -60,9 +60,9 @@ var sTestsOnStorIT = []func(t *testing.T){
 	testOnStorITCacheLCR,
 	testOnStorITCacheAlias,
 	testOnStorITCacheReverseAlias,
+	testOnStorITCacheResource,
 	testOnStorITCacheResourceCfg,
 	testOnStorITCacheTiming,
-	testOnStorITCacheResource,
 	// ToDo: test cache flush for a prefix
 	// ToDo: testOnStorITLoadAccountingCache
 	testOnStorITHasData,
@@ -84,9 +84,9 @@ var sTestsOnStorIT = []func(t *testing.T){
 	testOnStorITCRUDUser,
 	testOnStorITCRUDAlias,
 	testOnStorITCRUDReverseAlias,
+	testOnStorITCRUDResource,
 	testOnStorITCRUDResourceCfg,
 	testOnStorITCRUDTiming,
-	testOnStorITCRUDResource,
 	testOnStorITCRUDHistory,
 	testOnStorITCRUDStructVersion,
 	testOnStorITCRUDSQStoredMetrics,
@@ -851,7 +851,6 @@ func testOnStorITCacheTiming(t *testing.T) {
 	}
 }
 
-//test here Cache
 func testOnStorITCacheResource(t *testing.T) {
 	res := &Resource{
 		ID: "RL1",
@@ -1892,7 +1891,6 @@ func testOnStorITCRUDHistory(t *testing.T) {
 	}
 }
 
-//test here Crud
 func testOnStorITCRUDResource(t *testing.T) {
 	res := &Resource{
 		ID: "RL1",

engine/storage_interface.go

@@ -80,7 +80,7 @@ type DataDB interface {
 	RemAccountActionPlans(acntID string, apIDs []string) (err error)
 	PushTask(*Task) error
 	PopTask() (*Task, error)
-	LoadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) error
+	LoadAccountingCache(alsIDs, rvAlsIDs, rlIDs, resIDs []string) error
 	GetAccount(string) (*Account, error)
 	SetAccount(*Account) error
 	RemoveAccount(string) error

engine/storage_map.go

@@ -165,7 +165,7 @@ func (ms *MapStorage) LoadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs,
 	return
 }
 
-func (ms *MapStorage) LoadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) error {
+func (ms *MapStorage) LoadAccountingCache(alsIDs, rvAlsIDs, rlIDs, resIDs []string) error {
 	if ms.cacheCfg == nil {
 		return nil
 	}

engine/storage_mongo_datadb.go

@@ -1,4 +1,4 @@
-/* 
+/*
 Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
 Copyright (C) ITsysCOM GmbH
 
@@ -438,11 +438,12 @@ func (ms *MongoStorage) LoadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs
 	return
 }
 
-func (ms *MongoStorage) LoadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) (err error) {
+func (ms *MongoStorage) LoadAccountingCache(alsIDs, rvAlsIDs, rlIDs, resIDs []string) (err error) {
 	for key, ids := range map[string][]string{
 		utils.ALIASES_PREFIX:         alsIDs,
 		utils.REVERSE_ALIASES_PREFIX: rvAlsIDs,
 		utils.ResourceConfigsPrefix:  rlIDs,
+		utils.ResourcesPrefix:        resIDs,
 	} {
 		if err = ms.CacheDataFromDB(key, ids, false); err != nil {
 			return
@@ -1902,8 +1903,6 @@ func (ms *MongoStorage) RemoveResourceCfg(id string, transactionID string) (err
 	return nil
 }
 
-//from here
-//find the right collumn
 func (ms *MongoStorage) GetResource(id string, skipCache bool, transactionID string) (r *Resource, err error) {
 	key := utils.ResourcesPrefix + id
 	if !skipCache {
@@ -1945,8 +1944,6 @@ func (ms *MongoStorage) RemoveResource(id string, transactionID string) (err err
 	return nil
 }
 
-//to here
-
 func (ms *MongoStorage) GetTiming(id string, skipCache bool, transactionID string) (t *utils.TPTiming, err error) {
 	key := utils.TimingsPrefix + id
 	if !skipCache {

engine/storage_redis.go

@@ -1,4 +1,4 @@
-/* 
+/*
 Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
 Copyright (C) ITsysCOM GmbH
 
@@ -140,11 +140,12 @@ func (rs *RedisStorage) LoadRatingCache(dstIDs, rvDstIDs, rplIDs, rpfIDs, actIDs
 	return
 }
 
-func (rs *RedisStorage) LoadAccountingCache(alsIDs, rvAlsIDs, rlIDs []string) (err error) {
+func (rs *RedisStorage) LoadAccountingCache(alsIDs, rvAlsIDs, rlIDs, resIDs []string) (err error) {
 	for key, ids := range map[string][]string{
 		utils.ALIASES_PREFIX:         alsIDs,
 		utils.REVERSE_ALIASES_PREFIX: rvAlsIDs,
 		utils.ResourceConfigsPrefix:  rlIDs,
+		utils.ResourcesPrefix:        resIDs,
 	} {
 		if err = rs.CacheDataFromDB(key, ids, false); err != nil {
 			return
@@ -1424,7 +1425,6 @@ func (rs *RedisStorage) RemoveResourceCfg(id string, transactionID string) (err
 	return
 }
 
-//from here
 func (rs *RedisStorage) GetResource(id string, skipCache bool, transactionID string) (r *Resource, err error) {
 	key := utils.ResourcesPrefix + id
 	if !skipCache {
@@ -1467,8 +1467,6 @@ func (rs *RedisStorage) RemoveResource(id string, transactionID string) (err err
 	return
 }
 
-// to here
-
 func (rs *RedisStorage) GetTiming(id string, skipCache bool, transactionID string) (t *utils.TPTiming, err error) {
 	key := utils.TimingsPrefix + id
 	if !skipCache {

general_tests/acntacts_test.go

@@ -64,7 +64,7 @@ ENABLE_ACNT,*enable_account,,,,,,,,,,,,,,false,false,10`
 
 	cache.Flush()
 	dbAcntActs.LoadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
-	dbAcntActs.LoadAccountingCache(nil, nil, nil)
+	dbAcntActs.LoadAccountingCache(nil, nil, nil, nil)
 
 	expectAcnt := &engine.Account{ID: "cgrates.org:1"}
 	if acnt, err := dbAcntActs.GetAccount("cgrates.org:1"); err != nil {

general_tests/auth_test.go

@@ -76,7 +76,7 @@ RP_ANY,DR_ANY_1CNT,*any,10`
 
 	cache.Flush()
 	dbAuth.LoadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
-	dbAuth.LoadAccountingCache(nil, nil, nil)
+	dbAuth.LoadAccountingCache(nil, nil, nil, nil)
 
 	if cachedDests := cache.CountEntries(utils.DESTINATION_PREFIX); cachedDests != 0 {
 		t.Error("Wrong number of cached destinations found", cachedDests)

general_tests/costs1_test.go

@@ -74,7 +74,7 @@ RP_SMS1,DR_SMS_1,ALWAYS,10`
 	csvr.WriteToDatabase(false, false, false)
 	cache.Flush()
 	dataDB.LoadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
-	dataDB.LoadAccountingCache(nil, nil, nil)
+	dataDB.LoadAccountingCache(nil, nil, nil, nil)
 
 	if cachedRPlans := cache.CountEntries(utils.RATING_PLAN_PREFIX); cachedRPlans != 3 {
 		t.Error("Wrong number of cached rating plans found", cachedRPlans)

general_tests/datachrg1_test.go

@@ -61,7 +61,7 @@ RP_DATA1,DR_DATA_2,TM2,10`
 	csvr.WriteToDatabase(false, false, false)
 	cache.Flush()
 	dataDB.LoadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
-	dataDB.LoadAccountingCache(nil, nil, nil)
+	dataDB.LoadAccountingCache(nil, nil, nil, nil)
 
 	if cachedRPlans := cache.CountEntries(utils.RATING_PLAN_PREFIX); cachedRPlans != 1 {
 		t.Error("Wrong number of cached rating plans found", cachedRPlans)

general_tests/ddazmbl1_test.go

@@ -111,7 +111,7 @@ TOPUP10_AT,TOPUP10_AC1,ASAP,10`
 	}
 	cache.Flush()
 	dataDB.LoadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
-	dataDB.LoadAccountingCache(nil, nil, nil)
+	dataDB.LoadAccountingCache(nil, nil, nil, nil)
 
 	if cachedDests := cache.CountEntries(utils.DESTINATION_PREFIX); cachedDests != 0 {
 		t.Error("Wrong number of cached destinations found", cachedDests)

general_tests/ddazmbl2_test.go

@@ -111,7 +111,7 @@ TOPUP10_AT,TOPUP10_AC1,ASAP,10`
 	}
 	cache.Flush()
 	dataDB2.LoadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
-	dataDB2.LoadAccountingCache(nil, nil, nil)
+	dataDB2.LoadAccountingCache(nil, nil, nil, nil)
 
 	if cachedDests := cache.CountEntries(utils.DESTINATION_PREFIX); cachedDests != 0 {
 		t.Error("Wrong number of cached destinations found", cachedDests)

general_tests/ddazmbl3_test.go

@@ -109,7 +109,7 @@ RP_UK,DR_UK_Mobile_BIG5,ALWAYS,10`
 	}
 	cache.Flush()
 	dataDB3.LoadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
-	dataDB3.LoadAccountingCache(nil, nil, nil)
+	dataDB3.LoadAccountingCache(nil, nil, nil, nil)
 
 	if cachedDests := cache.CountEntries(utils.DESTINATION_PREFIX); cachedDests != 0 {
 		t.Error("Wrong number of cached destinations found", cachedDests)

general_tests/smschrg1_test.go

@@ -59,7 +59,7 @@ func TestSMSLoadCsvTpSmsChrg1(t *testing.T) {
 	csvr.WriteToDatabase(false, false, false)
 	cache.Flush()
 	dataDB.LoadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
-	dataDB.LoadAccountingCache(nil, nil, nil)
+	dataDB.LoadAccountingCache(nil, nil, nil, nil)
 
 	if cachedRPlans := cache.CountEntries(utils.RATING_PLAN_PREFIX); cachedRPlans != 1 {
 		t.Error("Wrong number of cached rating plans found", cachedRPlans)

general_tests/tutorial_it_test.go

@@ -104,7 +104,7 @@ func TestTutITCacheStats(t *testing.T) {
 	var rcvStats *utils.CacheStats
 	expectedStats := &utils.CacheStats{Destinations: 5, ReverseDestinations: 7, RatingPlans: 4, RatingProfiles: 9,
 		Actions: 8, ActionPlans: 4, AccountActionPlans: 5, SharedGroups: 1, DerivedChargers: 1, LcrProfiles: 5,
-		CdrStats: 6, Users: 3, Aliases: 1, ReverseAliases: 2, Resources: 3}
+		CdrStats: 6, Users: 3, Aliases: 1, ReverseAliases: 2, ResourcesConfig: 0, Resources: 3}
 	var args utils.AttrCacheStats
 	if err := tutLocalRpc.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil {
 		t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())

utils/apitpdata.go

@@ -667,6 +667,7 @@ type ArgsCache struct {
 	DerivedChargerIDs     *[]string
 	AliasIDs              *[]string
 	ReverseAliasIDs       *[]string
+	ResourceConfigIDs     *[]string
 	ResourceIDs           *[]string
 	StatsIDs              *[]string
 	ThresholdsIDs         *[]string
@@ -704,6 +705,7 @@ type CacheStats struct {
 	Users               int
 	Aliases             int
 	ReverseAliases      int
+	ResourcesConfig     int
 	Resources           int
 	//Stats               int
 	//thresholds		  int