From b1b8c111e29b457f3a732b03610506485d043e68 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 27 Jul 2015 13:59:56 +0300 Subject: [PATCH 1/7] optimized add alias apis --- apier/v1/aliases.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apier/v1/aliases.go b/apier/v1/aliases.go index 962949105..d0b276b45 100644 --- a/apier/v1/aliases.go +++ b/apier/v1/aliases.go @@ -45,7 +45,7 @@ func (self *ApierV1) AddRatingSubjectAliases(attrs AttrAddRatingSubjectAliases, } aliasesChanged = append(aliasesChanged, utils.RP_ALIAS_PREFIX+utils.RatingSubjectAliasKey(attrs.Tenant, alias)) } - if err := self.RatingDb.CachePrefixes(utils.RP_ALIAS_PREFIX); err != nil { + if err := self.RatingDb.CachePrefixValues(map[string][]string{utils.RP_ALIAS_PREFIX: aliasesChanged}); err != nil { return utils.NewErrServerError(err) } *reply = utils.OK @@ -94,7 +94,7 @@ func (self *ApierV1) AddAccountAliases(attrs AttrAddAccountAliases, reply *strin } aliasesChanged = append(aliasesChanged, utils.ACC_ALIAS_PREFIX+utils.AccountAliasKey(attrs.Tenant, alias)) } - if err := self.RatingDb.CachePrefixes(utils.ACC_ALIAS_PREFIX); err != nil { + if err := self.RatingDb.CachePrefixValues(map[string][]string{utils.ACC_ALIAS_PREFIX: aliasesChanged}); err != nil { return utils.NewErrServerError(err) } *reply = utils.OK From b8e43f14f2a9dbc164d14c3efbaea32bfd0714dd Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 27 Jul 2015 16:21:30 +0300 Subject: [PATCH 2/7] optimized alliases removal --- apier/v1/aliases.go | 10 ++++++---- engine/storage_redis.go | 18 +++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/apier/v1/aliases.go b/apier/v1/aliases.go index d0b276b45..e85f0019f 100644 --- a/apier/v1/aliases.go +++ b/apier/v1/aliases.go @@ -76,9 +76,10 @@ func (self *ApierV1) RemRatingSubjectAliases(tenantRatingSubject engine.TenantRa return utils.NewErrServerError(err) } - if err := self.RatingDb.CachePrefixes(utils.RP_ALIAS_PREFIX); err != nil { + // cache refresh not needed, synched in RemoveRpAliases + /*if err := self.RatingDb.CachePrefixes(utils.RP_ALIAS_PREFIX); err != nil { return utils.NewErrServerError(err) - } + }*/ *reply = utils.OK return nil } @@ -124,9 +125,10 @@ func (self *ApierV1) RemAccountAliases(tenantAccount engine.TenantAccount, reply if err := self.RatingDb.RemoveAccAliases([]*engine.TenantAccount{&tenantAccount}); err != nil { return utils.NewErrServerError(err) } - if err := self.RatingDb.CachePrefixes(utils.ACC_ALIAS_PREFIX); err != nil { + // cache refresh not needed, synched in RemoveRpAliases + /*if err := self.RatingDb.CachePrefixes(utils.ACC_ALIAS_PREFIX); err != nil { return utils.NewErrServerError(err) - } + }*/ *reply = utils.OK return nil } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 658b31d1e..062f2a018 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -411,15 +411,13 @@ func (rs *RedisStorage) SetRpAlias(key, alias string) (err error) { // Removes the aliases of a specific account, on a tenant func (rs *RedisStorage) RemoveRpAliases(tenantRtSubjects []*TenantRatingSubject) (err error) { - alsKeys, err := rs.db.Keys(utils.RP_ALIAS_PREFIX + "*") + alsMap, err := cache2go.GetAllEntries(utils.RP_ALIAS_PREFIX) if err != nil { return err } - for _, key := range alsKeys { - alias, err := rs.GetRpAlias(key[len(utils.RP_ALIAS_PREFIX):], true) - if err != nil { - return err - } + + for key, aliasInterface := range alsMap { + alias := aliasInterface.Value().(string) for _, tntRSubj := range tenantRtSubjects { tenantPrfx := utils.RP_ALIAS_PREFIX + tntRSubj.Tenant + utils.CONCATENATED_KEY_SEP if len(key) < len(tenantPrfx) || tenantPrfx != key[:len(tenantPrfx)] { // filter out the tenant for accounts @@ -428,10 +426,10 @@ func (rs *RedisStorage) RemoveRpAliases(tenantRtSubjects []*TenantRatingSubject) if tntRSubj.Subject != alias { continue } - cache2go.RemKey(key) if _, err = rs.db.Del(key); err != nil { return err } + cache2go.RemKey(key) break } } @@ -508,11 +506,13 @@ func (rs *RedisStorage) SetAccAlias(key, alias string) (err error) { } func (rs *RedisStorage) RemoveAccAliases(tenantAccounts []*TenantAccount) (err error) { - alsKeys, err := rs.db.Keys(utils.ACC_ALIAS_PREFIX + "*") + alsMap, err := cache2go.GetAllEntries(utils.ACC_ALIAS_PREFIX) if err != nil { return err } - for _, key := range alsKeys { + + for key, aliasInterface := range alsMap { + alias := aliasInterface.Value().(string) alias, err := rs.GetAccAlias(key[len(utils.ACC_ALIAS_PREFIX):], true) if err != nil { return err From ea4a6e6f6a9b8bca4017473440fd3c9105a584e3 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 27 Jul 2015 17:05:14 +0300 Subject: [PATCH 3/7] ceche for not found err --- apier/v1/aliases.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apier/v1/aliases.go b/apier/v1/aliases.go index e85f0019f..7825531a0 100644 --- a/apier/v1/aliases.go +++ b/apier/v1/aliases.go @@ -73,6 +73,9 @@ func (self *ApierV1) RemRatingSubjectAliases(tenantRatingSubject engine.TenantRa return utils.NewErrMandatoryIeMissing(missing...) } if err := self.RatingDb.RemoveRpAliases([]*engine.TenantRatingSubject{&tenantRatingSubject}); err != nil { + if err == utils.ErrNotFound { + return err + } return utils.NewErrServerError(err) } @@ -123,6 +126,9 @@ func (self *ApierV1) RemAccountAliases(tenantAccount engine.TenantAccount, reply return utils.NewErrMandatoryIeMissing(missing...) } if err := self.RatingDb.RemoveAccAliases([]*engine.TenantAccount{&tenantAccount}); err != nil { + if err == utils.ErrNotFound { + return err + } return utils.NewErrServerError(err) } // cache refresh not needed, synched in RemoveRpAliases From 78c358e45ba832adbeea2a49c597272847b60115 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 27 Jul 2015 18:19:05 +0300 Subject: [PATCH 4/7] fixess for alias removal --- engine/storage_redis.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 062f2a018..b19366a0c 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -23,6 +23,8 @@ import ( "compress/zlib" "errors" "fmt" + "log" + "strings" "github.com/cgrates/cgrates/cache2go" "github.com/cgrates/cgrates/utils" @@ -419,14 +421,14 @@ func (rs *RedisStorage) RemoveRpAliases(tenantRtSubjects []*TenantRatingSubject) for key, aliasInterface := range alsMap { alias := aliasInterface.Value().(string) for _, tntRSubj := range tenantRtSubjects { - tenantPrfx := utils.RP_ALIAS_PREFIX + tntRSubj.Tenant + utils.CONCATENATED_KEY_SEP - if len(key) < len(tenantPrfx) || tenantPrfx != key[:len(tenantPrfx)] { // filter out the tenant for accounts + tenantPrfx := tntRSubj.Tenant + utils.CONCATENATED_KEY_SEP + if len(key) < len(tenantPrfx) || !strings.HasPrefix(key, tenantPrfx) { // filter out the tenant for accounts continue } if tntRSubj.Subject != alias { continue } - if _, err = rs.db.Del(key); err != nil { + if _, err = rs.db.Del(utils.RP_ALIAS_PREFIX + key); err != nil { return err } cache2go.RemKey(key) @@ -513,22 +515,22 @@ func (rs *RedisStorage) RemoveAccAliases(tenantAccounts []*TenantAccount) (err e for key, aliasInterface := range alsMap { alias := aliasInterface.Value().(string) - alias, err := rs.GetAccAlias(key[len(utils.ACC_ALIAS_PREFIX):], true) - if err != nil { - return err - } + log.Print("Key: ", key) + log.Print("Alias: ", alias) for _, tntAcnt := range tenantAccounts { - tenantPrfx := utils.ACC_ALIAS_PREFIX + tntAcnt.Tenant + utils.CONCATENATED_KEY_SEP - if len(key) < len(tenantPrfx) || tenantPrfx != key[:len(tenantPrfx)] { // filter out the tenant for accounts + tenantPrfx := tntAcnt.Tenant + utils.CONCATENATED_KEY_SEP + log.Print("Tenant: ", tenantPrfx) + if len(key) < len(tenantPrfx) || !strings.HasPrefix(key, tenantPrfx) { // filter out the tenant for accounts continue } if tntAcnt.Account != alias { continue } - cache2go.RemKey(key) - if _, err = rs.db.Del(key); err != nil { + log.Print("DelKey: ", key) + if _, err = rs.db.Del(utils.ACC_ALIAS_PREFIX + key); err != nil { return err } + cache2go.RemKey(key) } } return From 520f82c27a6697e2b050b353de6e932e0d491f0d Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 27 Jul 2015 19:04:33 +0300 Subject: [PATCH 5/7] more aliases fixes --- engine/storage_redis.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/engine/storage_redis.go b/engine/storage_redis.go index b19366a0c..5f5648bda 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -23,7 +23,6 @@ import ( "compress/zlib" "errors" "fmt" - "log" "strings" "github.com/cgrates/cgrates/cache2go" @@ -431,7 +430,7 @@ func (rs *RedisStorage) RemoveRpAliases(tenantRtSubjects []*TenantRatingSubject) if _, err = rs.db.Del(utils.RP_ALIAS_PREFIX + key); err != nil { return err } - cache2go.RemKey(key) + cache2go.RemKey(utils.RP_ALIAS_PREFIX + key) break } } @@ -515,22 +514,18 @@ func (rs *RedisStorage) RemoveAccAliases(tenantAccounts []*TenantAccount) (err e for key, aliasInterface := range alsMap { alias := aliasInterface.Value().(string) - log.Print("Key: ", key) - log.Print("Alias: ", alias) for _, tntAcnt := range tenantAccounts { tenantPrfx := tntAcnt.Tenant + utils.CONCATENATED_KEY_SEP - log.Print("Tenant: ", tenantPrfx) if len(key) < len(tenantPrfx) || !strings.HasPrefix(key, tenantPrfx) { // filter out the tenant for accounts continue } if tntAcnt.Account != alias { continue } - log.Print("DelKey: ", key) if _, err = rs.db.Del(utils.ACC_ALIAS_PREFIX + key); err != nil { return err } - cache2go.RemKey(key) + cache2go.RemKey(utils.ACC_ALIAS_PREFIX + key) } } return From f3257b5dbac65aedb1f3b47a7a1319a3fdd32e06 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 27 Jul 2015 20:50:14 +0300 Subject: [PATCH 6/7] aliases tests --- engine/storage_test.go | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/engine/storage_test.go b/engine/storage_test.go index 4537406a9..21cf88006 100644 --- a/engine/storage_test.go +++ b/engine/storage_test.go @@ -153,9 +153,6 @@ func TestGetRPAliases(t *testing.T) { } func TestRemRSubjAliases(t *testing.T) { - if err := ratingStorage.SetRpAlias(utils.RatingSubjectAliasKey("cgrates.org", "2001"), "1001"); err != nil { - t.Error(err) - } if err := ratingStorage.SetRpAlias(utils.RatingSubjectAliasKey("cgrates.org", "2002"), "1001"); err != nil { t.Error(err) } @@ -177,6 +174,24 @@ func TestRemRSubjAliases(t *testing.T) { } } +func TestStorageRpAliases(t *testing.T) { + if _, err := ratingStorage.GetRpAlias("cgrates.org:1991", true); err == nil { + t.Error("Found alias before setting") + } + if err := ratingStorage.SetRpAlias(utils.RatingSubjectAliasKey("cgrates.org", "1991"), "1991"); err != nil { + t.Error(err) + } + if _, err := ratingStorage.GetRpAlias("cgrates.org:1991", true); err != nil { + t.Error("Alias not found after setting") + } + if err := ratingStorage.RemoveRpAliases([]*TenantRatingSubject{&TenantRatingSubject{Tenant: "cgrates.org", Subject: "1991"}}); err != nil { + t.Error(err) + } + if _, err := ratingStorage.GetRpAlias("cgrates.org:1991", true); err == nil { + t.Error("Found alias after removing") + } +} + func TestGetAccountAliases(t *testing.T) { if err := ratingStorage.SetAccAlias(utils.AccountAliasKey("cgrates.org", "2001"), "1001"); err != nil { t.Error(err) @@ -200,6 +215,24 @@ func TestGetAccountAliases(t *testing.T) { } } +func TestStorageAccAliases(t *testing.T) { + if _, err := ratingStorage.GetAccAlias("cgrates.org:1991", true); err == nil { + t.Error("Found alias before setting") + } + if err := ratingStorage.SetAccAlias(utils.AccountAliasKey("cgrates.org", "1991"), "1991"); err != nil { + t.Error(err) + } + if _, err := ratingStorage.GetAccAlias("cgrates.org:1991", true); err != nil { + t.Error("Alias not found after setting") + } + if err := ratingStorage.RemoveAccAliases([]*TenantAccount{&TenantAccount{Tenant: "cgrates.org", Account: "1991"}}); err != nil { + t.Error(err) + } + if _, err := ratingStorage.GetAccAlias("cgrates.org:1991", true); err == nil { + t.Error("Found alias after removing") + } +} + /************************** Benchmarks *****************************/ func GetUB() *Account { From 4203a49ac831c888f09e760c794fc330be5b6261 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 27 Jul 2015 21:16:39 +0300 Subject: [PATCH 7/7] better tests for max session time --- engine/calldesc.go | 1 + engine/calldesc_test.go | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/engine/calldesc.go b/engine/calldesc.go index beb3fdaa7..8024184b2 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -531,6 +531,7 @@ func (origCD *CallDescriptor) getMaxSessionDuration(origAcc *Account) (time.Dura // cc, err := cd.debit(account, true, false) //Logger.Debug("CC: " + utils.ToJSON(cc)) + //log.Print("CC: ", utils.ToIJSON(cc)) //Logger.Debug(fmt.Sprintf("ERR: %v", err)) if err != nil { return 0, err diff --git a/engine/calldesc_test.go b/engine/calldesc_test.go index 23a73a8cf..b17046165 100644 --- a/engine/calldesc_test.go +++ b/engine/calldesc_test.go @@ -860,8 +860,8 @@ func TestMaxSesionTimeEmptyBalanceAndNoCost(t *testing.T) { func TestMaxSesionTimeLong(t *testing.T) { cd := &CallDescriptor{ - TimeStart: time.Date(2015, 07, 26, 13, 37, 0, 0, time.UTC), - TimeEnd: time.Date(2015, 07, 26, 16, 37, 0, 0, time.UTC), + TimeStart: time.Date(2015, 07, 24, 13, 37, 0, 0, time.UTC), + TimeEnd: time.Date(2015, 07, 24, 15, 37, 0, 0, time.UTC), Direction: "*out", Category: "call", Tenant: "cgrates.org", @@ -877,8 +877,8 @@ func TestMaxSesionTimeLong(t *testing.T) { func TestMaxSesionTimeLongerThanMoney(t *testing.T) { cd := &CallDescriptor{ - TimeStart: time.Date(2015, 07, 26, 13, 37, 0, 0, time.UTC), - TimeEnd: time.Date(2015, 07, 26, 19, 37, 0, 0, time.UTC), + TimeStart: time.Date(2015, 07, 24, 13, 37, 0, 0, time.UTC), + TimeEnd: time.Date(2015, 07, 24, 16, 37, 0, 0, time.UTC), Direction: "*out", Category: "call", Tenant: "cgrates.org", @@ -887,7 +887,7 @@ func TestMaxSesionTimeLongerThanMoney(t *testing.T) { } acc, _ := accountingStorage.GetAccount("*out:cgrates.org:money") allowedTime, err := cd.getMaxSessionDuration(acc) - expected, err := time.ParseDuration("5h33m20s") + expected, err := time.ParseDuration("2h46m40s") if err != nil || allowedTime != expected { t.Errorf("Expected: %v got %v", expected, allowedTime) }