diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 6f141025a..4b9e95823 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -42,7 +42,7 @@ type ApierV1 struct { } func (self *ApierV1) GetDestination(dstId string, reply *engine.Destination) error { - if dst, err := self.DataDb.GetDestination(dstId, false); err != nil { + if dst, err := self.DataDb.GetDestination(dstId); err != nil { return errors.New(utils.ERR_NOT_FOUND) } else { *reply = *dst diff --git a/cache2go/cache.go b/cache2go/cache.go index 4aee5b2ef..b7ab7c47e 100644 --- a/cache2go/cache.go +++ b/cache2go/cache.go @@ -135,6 +135,16 @@ func RemKey(key string) { delete(cache, key) } +func RemPrefixKey(prefix string) { + mux.Lock() + defer mux.Unlock() + for key, _ := range cache { + if strings.HasPrefix(key, prefix) { + delete(cache, key) + } + } +} + func XRemKey(key string) { xMux.Lock() defer xMux.Unlock() @@ -145,6 +155,20 @@ func XRemKey(key string) { } delete(xcache, key) } +func XRemPrefixKey(prefix string) { + xMux.Lock() + defer xMux.Unlock() + for key, _ := range xcache { + if strings.HasPrefix(key, prefix) { + if r, ok := xcache[key]; ok { + if r.timer() != nil { + r.timer().Stop() + } + } + delete(xcache, key) + } + } +} // Delete all keys from expiraton cache func XFlush() { diff --git a/cache2go/cache_test.go b/cache2go/cache_test.go index f7313dae1..f6b28a733 100644 --- a/cache2go/cache_test.go +++ b/cache2go/cache_test.go @@ -113,3 +113,27 @@ func TestXGetKeyAge(t *testing.T) { t.Error("Error getting cache key age: ", d) } } + +func TestRemPrefixKey(t *testing.T) { + Cache("x_t1", "test") + Cache("y_t1", "test") + RemPrefixKey("x_") + _, errX := GetCached("x_t1") + _, errY := GetCached("y_t1") + if errX == nil || errY != nil { + t.Error("Error removing prefix: ", errX, errY) + } +} + +func TestXRemPrefixKey(t *testing.T) { + a := &myStruct{data: "mama are mere"} + a.XCache("x_t1", 10*time.Second, a) + a.XCache("y_t1", 10*time.Second, a) + + XRemPrefixKey("x_") + _, errX := GetXCached("x_t1") + _, errY := GetXCached("y_t1") + if errX == nil || errY != nil { + t.Error("Error removing prefix: ", errX, errY) + } +} diff --git a/cmd/stress/cgr-spansstress/cgr-spansstress.go b/cmd/stress/cgr-spansstress/cgr-spansstress.go index ea8539f9a..0635191c9 100644 --- a/cmd/stress/cgr-spansstress/cgr-spansstress.go +++ b/cmd/stress/cgr-spansstress/cgr-spansstress.go @@ -54,9 +54,9 @@ func main() { CallDuration: 60 * time.Second, Direction: "*out", TOR: "call", - Tenant: "cgrates.org", - Subject: "1001", - Destination: "+49676016500", + Tenant: "185.25.80.254", + Subject: "dan", + Destination: "+4986517174963", } getter, err := engine.ConfigureDataStorage(utils.REDIS, "127.0.0.1", "6379", "10", "", "", utils.MSGPACK) if err != nil { @@ -75,7 +75,7 @@ func main() { j := 0 start := time.Now() for i := 0; i < *runs; i++ { - result, err = cd.Debit() + result, err = cd.GetCost() if *memprofile != "" { runtime.MemProfileRate = 1 runtime.GC() diff --git a/engine/destinations.go b/engine/destinations.go index 5c73a81f4..e4e9162e1 100644 --- a/engine/destinations.go +++ b/engine/destinations.go @@ -18,23 +18,14 @@ along with this program. If not, see package engine -import ( - "strings" - - "github.com/cgrates/cgrates/utils" -) - -const ( - LONG_PREFIX_SLICE_LENGTH = 30 -) +import "strings" /* Structure that gathers multiple destination prefixes under a common id. */ type Destination struct { - Id string - Prefixes []string - longPrefixesMap map[string]interface{} + Id string + Prefixes []string } // returns prefix precision @@ -42,18 +33,9 @@ func (d *Destination) containsPrefix(prefix string) int { if d == nil { return 0 } - if d.Prefixes != nil { - for _, p := range d.Prefixes { - if strings.Index(prefix, p) == 0 { - return len(p) - } - } - } - if d.longPrefixesMap != nil { - for _, p := range utils.SplitPrefix(prefix) { - if _, found := d.longPrefixesMap[p]; found { - return len(p) - } + for _, p := range d.Prefixes { + if strings.Index(prefix, p) == 0 { + return len(p) } } return 0 @@ -61,15 +43,8 @@ func (d *Destination) containsPrefix(prefix string) int { func (d *Destination) String() (result string) { result = d.Id + ": " - if d.Prefixes != nil { - for _, k := range d.Prefixes { - result += k + ", " - } - } - if d.longPrefixesMap != nil { - for k, _ := range d.longPrefixesMap { - result += k + ", " - } + for _, k := range d.Prefixes { + result += k + ", " } result = strings.TrimRight(result, ", ") return result @@ -78,13 +53,3 @@ func (d *Destination) String() (result string) { func (d *Destination) AddPrefix(pfx string) { d.Prefixes = append(d.Prefixes, pfx) } - -func (d *Destination) OptimizePrefixes() { - if len(d.Prefixes) > LONG_PREFIX_SLICE_LENGTH { - d.longPrefixesMap = make(map[string]interface{}) - for _, p := range d.Prefixes { - d.longPrefixesMap[p] = nil - } - d.Prefixes = nil - } -} diff --git a/engine/destinations_test.go b/engine/destinations_test.go index f0218fd35..be1f8df29 100644 --- a/engine/destinations_test.go +++ b/engine/destinations_test.go @@ -20,7 +20,6 @@ package engine import ( "encoding/json" - "strconv" "github.com/cgrates/cgrates/cache2go" @@ -44,7 +43,7 @@ func TestDestinationStorageStore(t *testing.T) { if err != nil { t.Error("Error storing destination: ", err) } - result, err := storageGetter.GetDestination(nationale.Id, false) + result, err := storageGetter.GetDestination(nationale.Id) if nationale.containsPrefix("0257") == 0 || nationale.containsPrefix("0256") == 0 || nationale.containsPrefix("0723") == 0 { t.Errorf("Expected %q was %q", nationale, result) } @@ -75,61 +74,39 @@ func TestDestinationContainsPrefixWrong(t *testing.T) { } func TestDestinationGetExists(t *testing.T) { - d, err := storageGetter.GetDestination("NAT", false) + d, err := storageGetter.GetDestination("NAT") if err != nil || d == nil { t.Error("Could not get destination: ", d) } } func TestDestinationGetExistsCache(t *testing.T) { - storageGetter.GetDestination("NAT", false) - if _, err := cache2go.GetCached(DESTINATION_PREFIX + "NAT"); err != nil { + storageGetter.GetDestination("NAT") + if _, err := cache2go.GetCached(DESTINATION_PREFIX + "0256"); err != nil { t.Error("Destination not cached:", err) } } func TestDestinationGetNotExists(t *testing.T) { - d, err := storageGetter.GetDestination("not existing", false) + d, err := storageGetter.GetDestination("not existing") if d != nil { t.Error("Got false destination: ", d, err) } } func TestDestinationGetNotExistsCache(t *testing.T) { - storageGetter.GetDestination("not existing", false) + storageGetter.GetDestination("not existing") if d, err := cache2go.GetCached("not existing"); err == nil { t.Error("Bad destination cached: ", d) } } -func TestDestinationOptimzeShort(t *testing.T) { - d := &Destination{} - for i := 0; i < LONG_PREFIX_SLICE_LENGTH; i++ { - d.AddPrefix(strconv.Itoa(i)) - } - d.OptimizePrefixes() - if d.Prefixes == nil || d.longPrefixesMap != nil { - t.Logf("Error optimizing destinations %+v", d) - } -} - -func TestDestinationOptimzeLong(t *testing.T) { - d := &Destination{} - for i := 0; i < LONG_PREFIX_SLICE_LENGTH+1; i++ { - d.AddPrefix(strconv.Itoa(i)) - } - d.OptimizePrefixes() - if d.Prefixes != nil || d.longPrefixesMap == nil { - t.Logf("Error optimizing destinations %+v", d) - } -} - /********************************* Benchmarks **********************************/ func BenchmarkDestinationStorageStoreRestore(b *testing.B) { nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}} for i := 0; i < b.N; i++ { storageGetter.SetDestination(nationale) - storageGetter.GetDestination(nationale.Id, true) + storageGetter.GetDestination(nationale.Id) } } diff --git a/engine/ratingprofile.go b/engine/ratingprofile.go index 9aeec2131..ba80c1cd1 100644 --- a/engine/ratingprofile.go +++ b/engine/ratingprofile.go @@ -114,7 +114,7 @@ func (rp *RatingProfile) GetRatingPlansForPrefix(cd *CallDescriptor) (err error) bestPrecision := 0 var rps RateIntervalList for _, p := range utils.SplitPrefix(cd.Destination) { - if x, err := cache2go.GetCached(p); err == nil { + if x, err := cache2go.GetCached(DESTINATION_PREFIX + p); err == nil { destIds := x.([]string) for _, dId := range destIds { if _, ok := rpl.DestinationRates[dId]; ok { diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 9866afc7a..e59baa6d6 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -74,7 +74,7 @@ type DataStorage interface { SetRatingPlan(*RatingPlan) error GetRatingProfile(string, bool) (*RatingProfile, error) SetRatingProfile(*RatingProfile) error - GetDestination(string, bool) (*Destination, error) + GetDestination(string) (*Destination, error) // DestinationContainsPrefix(string, string) (int, error) SetDestination(*Destination) error GetActions(string, bool) (Actions, error) diff --git a/engine/storage_map.go b/engine/storage_map.go index 845347784..96051378f 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -49,11 +49,10 @@ func (ms *MapStorage) PreCache(dKeys, rpKeys, rpfKeys, actKeys []string) error { if dKeys == nil && rpKeys == nil && rpfKeys == nil && actKeys == nil { cache2go.Flush() } + cache2go.RemPrefixKey(DESTINATION_PREFIX) for k, _ := range ms.dict { if strings.HasPrefix(k, DESTINATION_PREFIX) { - cache2go.RemKey(k) - // TODO: here I must delete all optimized prefixes - if _, err := ms.GetDestination(k[len(DESTINATION_PREFIX):], true); err != nil { + if _, err := ms.GetDestination(k[len(DESTINATION_PREFIX):]); err != nil { return err } } @@ -148,28 +147,20 @@ func (ms *MapStorage) SetRatingProfile(rpf *RatingProfile) (err error) { return } -func (ms *MapStorage) GetDestination(key string, checkDb bool) (dest *Destination, err error) { +func (ms *MapStorage) GetDestination(key string) (dest *Destination, err error) { key = DESTINATION_PREFIX + key - if x, err := cache2go.GetCached(key); err == nil { - return x.(*Destination), nil - } - if !checkDb { - return nil, errors.New(utils.ERR_NOT_FOUND) - } if values, ok := ms.dict[key]; ok { dest = &Destination{Id: key} err = ms.ms.Unmarshal(values, dest) // create optimized structure for _, p := range dest.Prefixes { var ids []string - if x, err := cache2go.GetCached(p); err == nil { + if x, err := cache2go.GetCached(DESTINATION_PREFIX + p); err == nil { ids = x.([]string) } ids = append(ids, dest.Id) - cache2go.Cache(p, ids) + cache2go.Cache(DESTINATION_PREFIX+p, ids) } - dest.OptimizePrefixes() - cache2go.Cache(key, dest) } else { return nil, errors.New("not found") } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 93b14da8a..c0ddefe08 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -77,12 +77,12 @@ func (rs *RedisStorage) PreCache(dKeys, rpKeys, rpfKeys, actKeys []string) (err if dKeys, err = rs.db.Keys(DESTINATION_PREFIX + "*"); err != nil { return } - } else if len(dKeys) != 0 { + cache2go.RemPrefixKey(DESTINATION_PREFIX) + } else if len(rpKeys) != 0 { Logger.Info(fmt.Sprintf("Caching destinations: %v", dKeys)) } for _, key := range dKeys { - cache2go.RemKey(key) - if _, err = rs.GetDestination(key[len(DESTINATION_PREFIX):], true); err != nil { + if _, err = rs.GetDestination(key[len(DESTINATION_PREFIX):]); err != nil { return err } } @@ -222,14 +222,8 @@ func (rs *RedisStorage) SetRatingProfile(rpf *RatingProfile) (err error) { return } -func (rs *RedisStorage) GetDestination(key string, checkDb bool) (dest *Destination, err error) { +func (rs *RedisStorage) GetDestination(key string) (dest *Destination, err error) { key = DESTINATION_PREFIX + key - if x, err := cache2go.GetCached(key); err == nil { - return x.(*Destination), nil - } - if !checkDb { - return nil, errors.New(utils.ERR_NOT_FOUND) - } var values []byte if values, err = rs.db.Get(key); len(values) > 0 && err == nil { b := bytes.NewBuffer(values) @@ -247,14 +241,12 @@ func (rs *RedisStorage) GetDestination(key string, checkDb bool) (dest *Destinat // create optimized structure for _, p := range dest.Prefixes { var ids []string - if x, err := cache2go.GetCached(p); err == nil { + if x, err := cache2go.GetCached(DESTINATION_PREFIX + p); err == nil { ids = x.([]string) } ids = append(ids, dest.Id) - cache2go.Cache(p, ids) + cache2go.Cache(DESTINATION_PREFIX+p, ids) } - dest.OptimizePrefixes() - cache2go.Cache(key, dest) } else { return nil, errors.New("not found") } diff --git a/engine/storage_test.go b/engine/storage_test.go index bf91f0fd5..63adb5c72 100644 --- a/engine/storage_test.go +++ b/engine/storage_test.go @@ -72,7 +72,7 @@ func TestMsgpackTime(t *testing.T) { } func TestStorageDestinationContainsPrefixShort(t *testing.T) { - dest, err := storageGetter.GetDestination("NAT", false) + dest, err := storageGetter.GetDestination("NAT") precision := dest.containsPrefix("0723") if err != nil || precision != 4 { t.Error("Error finding prefix: ", err, precision) @@ -80,7 +80,7 @@ func TestStorageDestinationContainsPrefixShort(t *testing.T) { } func TestStorageDestinationContainsPrefixLong(t *testing.T) { - dest, err := storageGetter.GetDestination("NAT", false) + dest, err := storageGetter.GetDestination("NAT") precision := dest.containsPrefix("0723045326") if err != nil || precision != 4 { t.Error("Error finding prefix: ", err, precision) @@ -88,7 +88,7 @@ func TestStorageDestinationContainsPrefixLong(t *testing.T) { } func TestStorageDestinationContainsPrefixNotExisting(t *testing.T) { - dest, err := storageGetter.GetDestination("NAT", false) + dest, err := storageGetter.GetDestination("NAT") precision := dest.containsPrefix("072") if err != nil || precision != 0 { t.Error("Error finding prefix: ", err, precision) @@ -96,11 +96,11 @@ func TestStorageDestinationContainsPrefixNotExisting(t *testing.T) { } func TestPreCacheRefresh(t *testing.T) { - storageGetter.SetDestination(&Destination{"T11", []string{"0"}, nil}) - storageGetter.GetDestination("T11", false) - storageGetter.SetDestination(&Destination{"T11", []string{"1"}, nil}) + storageGetter.SetDestination(&Destination{"T11", []string{"0"}}) + storageGetter.GetDestination("T11") + storageGetter.SetDestination(&Destination{"T11", []string{"1"}}) storageGetter.PreCache(nil, nil, nil, nil) - d, err := storageGetter.GetDestination("T11", false) + d, err := storageGetter.GetDestination("T11") p := d.containsPrefix("1") if err != nil || p == 0 { t.Error("Error refreshing cache:", d) diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index db63d1bcf..f9635bf7d 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -115,7 +115,7 @@ func (self *TPCSVImporter) importDestinations(fn string) error { } continue } - dst := &Destination{record[0], []string{record[1]}, nil} + dst := &Destination{record[0], []string{record[1]}} if err := self.StorDb.SetTPDestination(self.TPid, dst); err != nil { if self.Verbose { log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) diff --git a/engine/units_counter.go b/engine/units_counter.go index c5ecee41b..bbc82c6fb 100644 --- a/engine/units_counter.go +++ b/engine/units_counter.go @@ -18,6 +18,11 @@ along with this program. If not, see package engine +import ( + "github.com/cgrates/cgrates/cache2go" + "github.com/cgrates/cgrates/utils" +) + // Amount of a trafic of a certain type type UnitsCounter struct { Direction string @@ -63,16 +68,20 @@ func (uc *UnitsCounter) addUnits(amount float64, prefix string) { if !mb.HasDestination() { continue } - dest, err := storageGetter.GetDestination(mb.DestinationId, false) - if err != nil { - Logger.Err("Counter: unknown destination: " + mb.DestinationId) - continue - } - precision := dest.containsPrefix(prefix) - if precision > 0 { - mb.Value += amount - counted = true - break + for _, p := range utils.SplitPrefix(prefix) { + if x, err := cache2go.GetCached(DESTINATION_PREFIX + p); err == nil { + destIds := x.([]string) + for _, dId := range destIds { + if dId == mb.DestinationId { + mb.Value += amount + counted = true + break + } + } + } + if counted { + break + } } } } diff --git a/engine/userbalance.go b/engine/userbalance.go index 8608b3d33..bf21d5bb2 100644 --- a/engine/userbalance.go +++ b/engine/userbalance.go @@ -22,6 +22,7 @@ import ( "errors" "time" + "github.com/cgrates/cgrates/cache2go" "github.com/cgrates/cgrates/utils" "strings" @@ -124,14 +125,20 @@ func (ub *UserBalance) getBalancesForPrefix(prefix string, balances BalanceChain continue } if b.DestinationId != "" && b.DestinationId != utils.ANY { - dest, err := storageGetter.GetDestination(b.DestinationId, false) - if err != nil { - continue - } - precision := dest.containsPrefix(prefix) - if precision > 0 { - b.precision = precision - usefulBalances = append(usefulBalances, b) + for _, p := range utils.SplitPrefix(prefix) { + if x, err := cache2go.GetCached(DESTINATION_PREFIX + p); err == nil { + destIds := x.([]string) + for _, dId := range destIds { + if dId == b.DestinationId { + b.precision = len(p) + usefulBalances = append(usefulBalances, b) + break + } + } + } + if b.precision > 0 { + break + } } } else { usefulBalances = append(usefulBalances, b)