uppal/cgrates
1
0
Fork 1
mirror of https://github.com/cgrates/cgrates.git synced 2026-02-15 05:09:54 +05:00
Code Issues Packages Projects Releases Wiki Activity

Updated cache reload

Browse Source
This commit is contained in:
Trial97
2021-05-06 18:14:10 +03:00
committed by Dan Christian Bogos
parent 9ce27430ca
commit c73556091f
14 changed files with 66 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/cgr-tester/cgr-tester.go
View File

@@ -91,7 +91,7 @@ func durInternalRater(cd *engine.CallDescriptorWithAPIOpts) (time.Duration, erro
dm := engine.NewDataManager(dbConn, cgrConfig.CacheCfg(), nil) // for the momentn we use here "" for sentinelName
defer dm.DataDB().Close()
engine.SetDataStorage(dm)
if err := dm.LoadDataDBCache(engine.GetDefaultEmptyArgCachePrefix()); err != nil {
if err := engine.LoadAllDataDBToCache(dm); err != nil {
return nilDuration, fmt.Errorf("Cache rating error: %s", err.Error())
}
log.Printf("Runnning %d cycles...", *runs)

2
engine/account.go
View File

@@ -248,7 +248,7 @@ func (acc *Account) debitBalanceAction(a *Action, reset, resetIfNegative bool) e
}
acc.BalanceMap[balanceType] = append(acc.BalanceMap[balanceType], bClone)
_, err := guardian.Guardian.Guard(func() (interface{}, error) {
sgs := make([]string, len(bClone.SharedGroups))
sgs := make([]string, len(bClone.SharedGroups)) // aici
i := 0
for sgID := range bClone.SharedGroups {
// add shared group member

40
engine/caches.go
View File

@@ -229,7 +229,8 @@ func (chS *CacheS) Precache() (err error) {
wg.Add(1)
go func(cacheID string) {
errCache := chS.dm.CacheDataFromDB(
utils.CacheInstanceToPrefix[cacheID], nil,
utils.CacheInstanceToPrefix[cacheID],
[]string{utils.MetaAny},
false)
if errCache != nil {
errChan <- fmt.Errorf("precaching cacheID <%s>, got error: %s", cacheID, errCache)
@@ -360,6 +361,14 @@ func (chS *CacheS) V1RemoveGroup(args *utils.ArgsGetGroupWithAPIOpts,
}
func (chS *CacheS) V1ReloadCache(attrs utils.AttrReloadCacheWithAPIOpts, reply *string) (err error) {
return chS.cacheDataFromDB(attrs, reply, true)
}
func (chS *CacheS) V1LoadCache(attrs utils.AttrReloadCacheWithAPIOpts, reply *string) (err error) {
return chS.cacheDataFromDB(attrs, reply, false)
}
func (chS *CacheS) cacheDataFromDB(attrs utils.AttrReloadCacheWithAPIOpts, reply *string, mustBeCached bool) (err error) {
for key, ids := range attrs.ArgsCache {
if prfx, has := utils.ArgCacheToPrefix[key]; has {
if err = chS.dm.CacheDataFromDB(prfx, ids, true); err != nil {
@@ -379,35 +388,8 @@ func (chS *CacheS) V1ReloadCache(attrs utils.AttrReloadCacheWithAPIOpts, reply *
chS.tCache.Set(utils.CacheLoadIDs, key, val, nil,
cacheCommit(utils.NonTransactional), utils.NonTransactional)
}
*reply = utils.OK
return nil
}
func (chS *CacheS) V1LoadCache(attrs utils.AttrReloadCacheWithAPIOpts, reply *string) (err error) {
args := make(map[string][]string)
for key, ids := range attrs.ArgsCache {
if prfx, has := utils.ArgCacheToPrefix[key]; has {
args[prfx] = ids
}
}
if err = chS.dm.LoadDataDBCache(args); err != nil {
return utils.NewErrServerError(err)
}
//get loadIDs for all types
var loadIDs map[string]int64
if loadIDs, err = chS.dm.GetItemLoadIDs(utils.EmptyString, false); err != nil {
if err != utils.ErrNotFound { // we can receive cache reload from LoaderS and we store the LoadID only after all Items was processed
return
}
loadIDs = make(map[string]int64)
}
for key, val := range populateCacheLoadIDs(loadIDs, attrs.ArgsCache) {
chS.tCache.Set(utils.CacheLoadIDs, key, val, nil,
cacheCommit(utils.NonTransactional), utils.NonTransactional)
}
*reply = utils.OK
return nil
return
}
//populateCacheLoadIDs populate cacheLoadIDs based on attrs

17
engine/datamanager.go
View File

@@ -101,21 +101,6 @@ func (dm *DataManager) DataDB() DataDB {
return nil
}
func (dm *DataManager) LoadDataDBCache(attr map[string][]string) (err error) {
if dm == nil {
return utils.ErrNoDatabaseConn
}
if dm.DataDB().GetStorageType() == utils.INTERNAL {
return // all the data is in cache already
}
for key, ids := range attr {
if err = dm.CacheDataFromDB(key, ids, false); err != nil {
return
}
}
return
}
func (dm *DataManager) CacheDataFromDB(prfx string, ids []string, mustBeCached bool) (err error) {
if dm == nil {
return utils.ErrNoDatabaseConn
@@ -131,7 +116,7 @@ func (dm *DataManager) CacheDataFromDB(prfx string, ids []string, mustBeCached b
}
if prfx == utils.MetaAPIBan { // no need for ids in this case
ids = []string{utils.EmptyString}
} else if ids == nil {
} else if len(ids) != 0 && ids[0] == utils.MetaAny {
if mustBeCached {
ids = Cache.GetItemIDs(utils.CachePrefixToInstance[prfx], utils.EmptyString)
} else {

75
engine/libtest.go
View File

@@ -562,37 +562,48 @@ func GetDefaultEmptyCacheStats() map[string]*ltcache.CacheStats {
}
}
func GetDefaultEmptyArgCachePrefix() map[string][]string {
return map[string][]string{
utils.DestinationPrefix: nil,
utils.ReverseDestinationPrefix: nil,
utils.RatingPlanPrefix: nil,
utils.RatingProfilePrefix: nil,
utils.ActionPrefix: nil,
utils.ActionPlanPrefix: nil,
utils.AccountActionPlansPrefix: nil,
utils.ActionTriggerPrefix: nil,
utils.SharedGroupPrefix: nil,
utils.ResourceProfilesPrefix: nil,
utils.ResourcesPrefix: nil,
utils.StatQueuePrefix: nil,
utils.StatQueueProfilePrefix: nil,
utils.ThresholdPrefix: nil,
utils.ThresholdProfilePrefix: nil,
utils.FilterPrefix: nil,
utils.RouteProfilePrefix: nil,
utils.AttributeProfilePrefix: nil,
utils.ChargerProfilePrefix: nil,
utils.DispatcherProfilePrefix: nil,
utils.DispatcherHostPrefix: nil,
utils.TimingsPrefix: nil,
utils.AttributeFilterIndexes: nil,
utils.ResourceFilterIndexes: nil,
utils.StatFilterIndexes: nil,
utils.ThresholdFilterIndexes: nil,
utils.RouteFilterIndexes: nil,
utils.ChargerFilterIndexes: nil,
utils.DispatcherFilterIndexes: nil,
utils.FilterIndexPrfx: nil,
func LoadAllDataDBToCache(dm *DataManager) (err error) {
if dm == nil {
return utils.ErrNoDatabaseConn
}
if dm.DataDB().GetStorageType() == utils.INTERNAL {
return // all the data is in cache already
}
for key, ids := range map[string][]string{
utils.DestinationPrefix: {utils.MetaAny},
utils.ReverseDestinationPrefix: {utils.MetaAny},
utils.RatingPlanPrefix: {utils.MetaAny},
utils.RatingProfilePrefix: {utils.MetaAny},
utils.ActionPrefix: {utils.MetaAny},
utils.ActionPlanPrefix: {utils.MetaAny},
utils.AccountActionPlansPrefix: {utils.MetaAny},
utils.ActionTriggerPrefix: {utils.MetaAny},
utils.SharedGroupPrefix: {utils.MetaAny},
utils.ResourceProfilesPrefix: {utils.MetaAny},
utils.ResourcesPrefix: {utils.MetaAny},
utils.StatQueuePrefix: {utils.MetaAny},
utils.StatQueueProfilePrefix: {utils.MetaAny},
utils.ThresholdPrefix: {utils.MetaAny},
utils.ThresholdProfilePrefix: {utils.MetaAny},
utils.FilterPrefix: {utils.MetaAny},
utils.RouteProfilePrefix: {utils.MetaAny},
utils.AttributeProfilePrefix: {utils.MetaAny},
utils.ChargerProfilePrefix: {utils.MetaAny},
utils.DispatcherProfilePrefix: {utils.MetaAny},
utils.DispatcherHostPrefix: {utils.MetaAny},
utils.TimingsPrefix: {utils.MetaAny},
utils.AttributeFilterIndexes: {utils.MetaAny},
utils.ResourceFilterIndexes: {utils.MetaAny},
utils.StatFilterIndexes: {utils.MetaAny},
utils.ThresholdFilterIndexes: {utils.MetaAny},
utils.RouteFilterIndexes: {utils.MetaAny},
utils.ChargerFilterIndexes: {utils.MetaAny},
utils.DispatcherFilterIndexes: {utils.MetaAny},
utils.FilterIndexPrfx: {utils.MetaAny},
} {
if err = dm.CacheDataFromDB(key, ids, false); err != nil {
return
}
}
return
}

2
engine/storage_test.go
View File

@@ -102,7 +102,7 @@ func TestStorageCacheRefresh(t *testing.T) {
dm.GetDestination("T11", false, utils.NonTransactional)
dm.SetDestination(&Destination{"T11", []string{"1"}}, utils.NonTransactional)
t.Log("Test cache refresh")
err := dm.LoadDataDBCache(GetDefaultEmptyArgCachePrefix())
err := LoadAllDataDBCache(dm)
if err != nil {
t.Error("Error cache rating: ", err)
}

3
general_tests/acntacts_test.go
View File

@@ -66,8 +66,7 @@ ENABLE_ACNT,*enable_account,,,,,,,,,,,,,false,false,10`
t.Fatal(err)
}
csvr.WriteToDatabase(false, false)
dbAcntActs.LoadDataDBCache(engine.GetDefaultEmptyArgCachePrefix())
engine.LoadAllDataDBToCache(dbAcntActs)
expectAcnt := &engine.Account{ID: "cgrates.org:1"}
if acnt, err := dbAcntActs.GetAccount("cgrates.org:1"); err != nil {

3
general_tests/auth_test.go
View File

@@ -107,8 +107,7 @@ cgrates.org,call,*any,2013-01-06T00:00:00Z,RP_ANY,`
} else if acnt == nil {
t.Error("No account saved")
}
dbAuth.LoadDataDBCache(engine.GetDefaultEmptyArgCachePrefix())
engine.LoadAllDataDBToCache(dbAuth)
if cachedDests := len(engine.Cache.GetItemIDs(utils.CacheDestinations, "")); cachedDests != 1 {
t.Error("Wrong number of cached destinations found", cachedDests)

2
general_tests/costs1_test.go
View File

@@ -83,7 +83,7 @@ cgrates.org,sms,*any,2012-01-01T00:00:00Z,RP_SMS1,`
t.Fatal(err)
}
csvr.WriteToDatabase(false, false)
dataDB.LoadDataDBCache(engine.GetDefaultEmptyArgCachePrefix())
engine.LoadAllDataDBToCache(dataDB)
if cachedRPlans := len(engine.Cache.GetItemIDs(utils.CacheRatingPlans, utils.EmptyString)); cachedRPlans != 3 {
t.Error("Wrong number of cached rating plans found", cachedRPlans)

2
general_tests/datachrg1_test.go
View File

@@ -68,7 +68,7 @@ RP_DATA1,DR_DATA_2,TM2,10`
t.Fatal(err)
}
csvr.WriteToDatabase(false, false)
dataDB.LoadDataDBCache(engine.GetDefaultEmptyArgCachePrefix())
engine.LoadAllDataDBToCache(dataDB)
if cachedRPlans := len(engine.Cache.GetItemIDs(utils.CacheRatingPlans, utils.EmptyString)); cachedRPlans != 1 {
t.Error("Wrong number of cached rating plans found", cachedRPlans)

2
general_tests/ddazmbl1_test.go
View File

@@ -112,7 +112,7 @@ TOPUP10_AT,TOPUP10_AC1,ASAP,10`
t.Error("No account saved")
}
dataDB.LoadDataDBCache(engine.GetDefaultEmptyArgCachePrefix())
engine.LoadAllDataDBToCache(dataDB)
if cachedDests := len(engine.Cache.GetItemIDs(utils.CacheDestinations, "")); cachedDests != 1 {
t.Error("Wrong number of cached destinations found", cachedDests)

2
general_tests/ddazmbl2_test.go
View File

@@ -109,7 +109,7 @@ TOPUP10_AT,TOPUP10_AC1,ASAP,10`
} else if acnt == nil {
t.Error("No account saved")
}
dataDB2.LoadDataDBCache(engine.GetDefaultEmptyArgCachePrefix())
engine.LoadAllDataDBToCache(dataDB2)
if cachedDests := len(engine.Cache.GetItemIDs(utils.CacheDestinations, "")); cachedDests != 1 {
t.Error("Wrong number of cached destinations found", cachedDests)

2
general_tests/ddazmbl3_test.go
View File

@@ -108,7 +108,7 @@ cgrates.org,call,discounted_minutes,2013-01-06T00:00:00Z,RP_UK_Mobile_BIG5_PKG,`
} else if acnt == nil {
t.Error("No account saved")
}
dataDB3.LoadDataDBCache(engine.GetDefaultEmptyArgCachePrefix())
engine.LoadAllDataDBToCache(dataDB3)
if cachedDests := len(engine.Cache.GetItemIDs(utils.CacheDestinations, "")); cachedDests != 1 {
t.Error("Wrong number of cached destinations found", cachedDests)

2
general_tests/smschrg1_test.go
View File

@@ -68,7 +68,7 @@ func TestSMSLoadCsvTpSmsChrg1(t *testing.T) {
t.Fatal(err)
}
csvr.WriteToDatabase(false, false)
dataDB.LoadDataDBCache(engine.GetDefaultEmptyArgCachePrefix())
engine.LoadAllDataDBToCache(dataDB)
if cachedRPlans := len(engine.Cache.GetItemIDs(utils.CacheRatingPlans, utils.EmptyString)); cachedRPlans != 1 {
t.Error("Wrong number of cached rating plans found", cachedRPlans)