diff --git a/apier/v1/apier.go b/apier/v1/apier.go index fe83aafb1..0a04b5f08 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -22,7 +22,6 @@ import ( "encoding/json" "errors" "fmt" - "log" "path" "github.com/cgrates/cgrates/cache2go" @@ -793,11 +792,9 @@ func (self *ApierV1) ReloadCache(attrs utils.ApiReloadCache, reply *string) erro dcsKeys[idx] = engine.DERIVEDCHARGERS_PREFIX + dc } } - log.Print("Cache Rating") if err := self.RatingDb.CacheRating(dstKeys, rpKeys, rpfKeys, rpAlsKeys, lcrKeys); err != nil { return err } - log.Print("Cache Accounting") if err := self.AccountDb.CacheAccounting(actKeys, shgKeys, accAlsKeys, dcsKeys); err != nil { return err } diff --git a/cache2go/cache.go b/cache2go/cache.go index 711b02125..43b0067bf 100644 --- a/cache2go/cache.go +++ b/cache2go/cache.go @@ -158,7 +158,7 @@ func Flush() { } } -func CountEntries(prefix string) (result int64) { +func CountEntries(prefix string) (result int) { mux.RLock() defer mux.RUnlock() return cache.CountEntriesForPrefix(prefix) diff --git a/cache2go/cache_test.go b/cache2go/cache_test.go index 7b915ae5a..1bae5050b 100644 --- a/cache2go/cache_test.go +++ b/cache2go/cache_test.go @@ -89,7 +89,7 @@ func TestCachePush(t *testing.T) { CachePush("ccc_t1", "1") CachePush("ccc_t1", "2") v, err := GetCached("ccc_t1") - if err != nil || len(v.([]interface{})) != 2 { + if err != nil || len(v.(map[interface{}]struct{})) != 2 { t.Error("Error in cache push: ", v) } } diff --git a/cache2go/store.go b/cache2go/store.go index a2148ad33..0c6fd9880 100644 --- a/cache2go/store.go +++ b/cache2go/store.go @@ -16,7 +16,7 @@ type cacheStore interface { GetAge(string) (time.Duration, error) Delete(string) DeletePrefix(string) - CountEntriesForPrefix(string) int64 + CountEntriesForPrefix(string) int GetAllForPrefix(string) (map[string]timestampedValue, error) GetKeysForPrefix(string) []string } @@ -37,21 +37,15 @@ func (cs cacheDoubleStore) Put(key string, value interface{}) { } func (cs cacheDoubleStore) Append(key string, value interface{}) { - var elements []interface{} - v, err := cs.Get(key) - if err == nil { - elements = v.([]interface{}) + var elements map[interface{}]struct{} // using map for faster check if element is present + if v, err := cs.Get(key); err == nil { + elements = v.(map[interface{}]struct{}) + } else { + elements = make(map[interface{}]struct{}) } // check if the val is already present - found := false - for _, v := range elements { - if value == v { - found = true - break - } - } - if !found { - elements = append(elements, value) + if _, found := elements[value]; !found { + elements[value] = struct{}{} } cache.Put(key, elements) } @@ -87,9 +81,9 @@ func (cs cacheDoubleStore) DeletePrefix(prefix string) { delete(cs, prefix) } -func (cs cacheDoubleStore) CountEntriesForPrefix(prefix string) int64 { +func (cs cacheDoubleStore) CountEntriesForPrefix(prefix string) int { if _, ok := cs[prefix]; ok { - return int64(len(cs[prefix])) + return len(cs[prefix]) } return 0 } @@ -116,13 +110,13 @@ func (cs cacheDoubleStore) GetKeysForPrefix(prefix string) (keys []string) { // faster to access type cacheSimpleStore struct { cache map[string]timestampedValue - counters map[string]int64 + counters map[string]int } func newSimpleStore() cacheSimpleStore { return cacheSimpleStore{ cache: make(map[string]timestampedValue), - counters: make(map[string]int64), + counters: make(map[string]int), } } @@ -135,22 +129,17 @@ func (cs cacheSimpleStore) Put(key string, value interface{}) { } func (cs cacheSimpleStore) Append(key string, value interface{}) { - var elements []interface{} - if ti, exists := cs.cache[key]; exists { - elements = ti.value.([]interface{}) + var elements map[interface{}]struct{} + if v, err := cs.Get(key); err == nil { + elements = v.(map[interface{}]struct{}) + } else { + elements = make(map[interface{}]struct{}) } // check if the val is already present - found := false - for _, v := range elements { - if value == v { - found = true - break - } + if _, found := elements[value]; !found { + elements[value] = struct{}{} } - if !found { - elements = append(elements, value) - } - cs.Put(key, elements) + cache.Put(key, elements) } func (cs cacheSimpleStore) Get(key string) (interface{}, error) { @@ -209,7 +198,7 @@ func (cs cacheSimpleStore) descount(key string) { } } -func (cs cacheSimpleStore) CountEntriesForPrefix(prefix string) int64 { +func (cs cacheSimpleStore) CountEntriesForPrefix(prefix string) int { if _, ok := cs.counters[prefix]; ok { return cs.counters[prefix] } diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 4e5cafceb..da112849c 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -22,10 +22,8 @@ import ( "errors" "flag" "fmt" - "io/ioutil" "log" "os" - "runtime" //"runtime" "strconv" "time" @@ -304,22 +302,7 @@ func writePid() { } } -func start() { - defer func() { - if r := recover(); r != nil { - engine.Logger.Crit(fmt.Sprintf("CRITICAL ERROR: %v", r)) - var stack [8192]byte - runtime.Stack(stack[:], false) - if tmpFile, err := ioutil.TempFile(os.TempDir(), "cgr_coredump"); err != nil { - engine.Logger.Crit(fmt.Sprintf("Cannot create coredump file: %v", err)) - engine.Logger.Crit(string(stack[:])) - } else { - tmpFile.Write(stack[:]) - tmpFile.Close() - engine.Logger.Crit(fmt.Sprintf("Core dumped: %s", tmpFile.Name())) - } - } - }() +func main() { flag.Parse() if *version { fmt.Println("CGRateS " + utils.VERSION) @@ -529,12 +512,3 @@ func start() { } engine.Logger.Info("Stopped all components. CGRateS shutdown!") } - -func main() { - defer func() { - if e := recover(); e != nil { - fmt.Fprintf(os.Stderr, "PANIC: %v\n", e) - } - }() - start() -} diff --git a/console/status.go b/console/status.go index e58151f5d..f316a4765 100644 --- a/console/status.go +++ b/console/status.go @@ -50,7 +50,7 @@ func (self *CmdStatus) RpcParams() interface{} { } func (self *CmdStatus) RpcResult() interface{} { - var s string + var s map[string]interface{} return &s } diff --git a/data/scripts/pkg/debian/cgrates.init b/data/scripts/pkg/debian/cgrates.init index 488c8b94f..e2417f293 100755 --- a/data/scripts/pkg/debian/cgrates.init +++ b/data/scripts/pkg/debian/cgrates.init @@ -23,6 +23,7 @@ GROUP=cgrates ENGINE_OPTS="" RUNDIR=/var/run/$NAME PIDFILE=$RUNDIR/cgr-engine.pid +COREDUMP=$RUNDIR/cgr-coredump.log SCRIPTNAME=/etc/init.d/$NAME DEFAULTS=/etc/default/$NAME ENABLE=false @@ -62,7 +63,8 @@ do_start() # 0 if daemon has been started # 1 if daemon was already running # 2 if daemon could not be started - start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --test > /dev/null \ + echo "Started at:" `date`>>$COREDUMP + start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --test > /dev/null 2>>$COREDUMP \ || return 1 start-stop-daemon --start --quiet --pidfile $PIDFILE --make-pidfile --exec $DAEMON --chuid $USER:$GROUP --background -- \ $ENGINE_OPTS \ diff --git a/engine/account.go b/engine/account.go index a089c59ac..1f6451b5d 100644 --- a/engine/account.go +++ b/engine/account.go @@ -160,8 +160,8 @@ func (ub *Account) getBalancesForPrefix(prefix, category string, balances Balanc if b.DestinationId != "" && b.DestinationId != utils.ANY { for _, p := range utils.SplitPrefix(prefix, MIN_PREFIX_MATCH) { if x, err := cache2go.GetCached(DESTINATION_PREFIX + p); err == nil { - destIds := x.([]interface{}) - for _, dId := range destIds { + destIds := x.(map[interface{}]struct{}) + for dId, _ := range destIds { if dId == b.DestinationId { b.precision = len(p) usefulBalances = append(usefulBalances, b) diff --git a/engine/destinations.go b/engine/destinations.go index 993ff301f..7af1d826e 100644 --- a/engine/destinations.go +++ b/engine/destinations.go @@ -23,7 +23,6 @@ import ( "strings" "github.com/cgrates/cgrates/cache2go" - "github.com/cgrates/cgrates/utils" "github.com/cgrates/cgrates/history" ) @@ -75,11 +74,8 @@ func (d *Destination) GetHistoryRecord() history.Record { // Reverse search in cache to see if prefix belongs to destination id func CachedDestHasPrefix(destId, prefix string) bool { if cached, err := cache2go.GetCached(DESTINATION_PREFIX + prefix); err == nil { - for _, cachedDstId := range cached.([]interface{}) { - if destId == cachedDstId { - return true - } - } + _, found := cached.(map[interface{}]struct{})[destId] + return found } return false } @@ -90,17 +86,16 @@ func CleanStalePrefixes(destIds []string) { return } for prefix, idIDs := range prefixMap { - dIDs := idIDs.Value().([]interface{}) - strdIDs := utils.ConvertInterfaceSliceToStringMap(dIDs) + dIDs := idIDs.Value().(map[interface{}]struct{}) changed := false for _, searchedDID := range destIds { - if i, found := strdIDs[searchedDID]; found { + if _, found := dIDs[searchedDID]; found { if len(dIDs) == 1 { // remove de prefix from cache cache2go.RemKey(DESTINATION_PREFIX + prefix) } else { // delete the destination from list and put the new list in chache - dIDs[i], dIDs = dIDs[len(dIDs)-1], dIDs[:len(dIDs)-1] + delete(dIDs, searchedDID) changed = true } } diff --git a/engine/destinations_test.go b/engine/destinations_test.go index 9eed163b4..31b9515e3 100644 --- a/engine/destinations_test.go +++ b/engine/destinations_test.go @@ -126,17 +126,18 @@ func TestNonCachedDestWrongPrefix(t *testing.T) { } func TestCleanStalePrefixes(t *testing.T) { - cache2go.Cache(DESTINATION_PREFIX+"1", []interface{}{"D1", "D2"}) - cache2go.Cache(DESTINATION_PREFIX+"2", []interface{}{"D1"}) - cache2go.Cache(DESTINATION_PREFIX+"3", []interface{}{"D2"}) + x := struct{}{} + cache2go.Cache(DESTINATION_PREFIX+"1", map[interface{}]struct{}{"D1": x, "D2": x}) + cache2go.Cache(DESTINATION_PREFIX+"2", map[interface{}]struct{}{"D1": x}) + cache2go.Cache(DESTINATION_PREFIX+"3", map[interface{}]struct{}{"D2": x}) CleanStalePrefixes([]string{"D1"}) - if r, err := cache2go.GetCached(DESTINATION_PREFIX + "1"); err != nil || len(r.([]interface{})) != 1 { + if r, err := cache2go.GetCached(DESTINATION_PREFIX + "1"); err != nil || len(r.(map[interface{}]struct{})) != 1 { t.Error("Error cleaning stale destination ids", r) } if r, err := cache2go.GetCached(DESTINATION_PREFIX + "2"); err == nil { t.Error("Error removing stale prefix: ", r) } - if r, err := cache2go.GetCached(DESTINATION_PREFIX + "3"); err != nil || len(r.([]interface{})) != 1 { + if r, err := cache2go.GetCached(DESTINATION_PREFIX + "3"); err != nil || len(r.(map[interface{}]struct{})) != 1 { t.Error("Error performing stale cleaning: ", r) } } diff --git a/engine/ratingprofile.go b/engine/ratingprofile.go index 85a877823..c667ab266 100644 --- a/engine/ratingprofile.go +++ b/engine/ratingprofile.go @@ -127,8 +127,8 @@ func (rp *RatingProfile) GetRatingPlansForPrefix(cd *CallDescriptor) (err error) } else { for _, p := range utils.SplitPrefix(cd.Destination, MIN_PREFIX_MATCH) { if x, err := cache2go.GetCached(DESTINATION_PREFIX + p); err == nil { - destIds := x.([]interface{}) - for _, idId := range destIds { + destIds := x.(map[interface{}]struct{}) + for idId, _ := range destIds { dId := idId.(string) if _, ok := rpl.DestinationRates[dId]; ok { rps = rpl.RateIntervalList(dId) diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 71a31615f..c86ec36a2 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -57,6 +57,8 @@ const ( CREATE_CDRS_TABLES_SQL = "create_cdrs_tables.sql" CREATE_TARIFFPLAN_TABLES_SQL = "create_tariffplan_tables.sql" TEST_SQL = "TEST_SQL" + + DESTINATIONS_LOAD_THRESHOLD = 0.1 ) type Storage interface { diff --git a/engine/storage_map.go b/engine/storage_map.go index 55db2969f..813d52215 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -58,7 +58,7 @@ func (ms *MapStorage) GetKeysForPrefix(string) ([]string, error) { func (ms *MapStorage) CacheRating(dKeys, rpKeys, rpfKeys, alsKeys, lcrKeys []string) error { cache2go.BeginTransaction() - if dKeys == nil { + if dKeys == nil || (float64(cache2go.CountEntries(DESTINATION_PREFIX))*DESTINATIONS_LOAD_THRESHOLD < float64(len(dKeys))) { cache2go.RemPrefixKey(DESTINATION_PREFIX) } else { CleanStalePrefixes(dKeys) diff --git a/engine/storage_redis.go b/engine/storage_redis.go index ec6d14b67..418d6a7ba 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -73,7 +73,8 @@ func (rs *RedisStorage) GetKeysForPrefix(prefix string) ([]string, error) { func (rs *RedisStorage) CacheRating(dKeys, rpKeys, rpfKeys, alsKeys, lcrKeys []string) (err error) { cache2go.BeginTransaction() - if dKeys == nil { + if dKeys == nil || (float64(cache2go.CountEntries(DESTINATION_PREFIX))*DESTINATIONS_LOAD_THRESHOLD < float64(len(dKeys))) { + // if need to load more than a half of exiting keys load them all Logger.Info("Caching all destinations") if dKeys, err = rs.db.Keys(DESTINATION_PREFIX + "*"); err != nil { cache2go.RollbackTransaction() diff --git a/engine/units_counter.go b/engine/units_counter.go index ab975a28f..8c192c27c 100644 --- a/engine/units_counter.go +++ b/engine/units_counter.go @@ -78,13 +78,11 @@ func (uc *UnitsCounter) addUnits(amount float64, prefix string) { } for _, p := range utils.SplitPrefix(prefix, MIN_PREFIX_MATCH) { if x, err := cache2go.GetCached(DESTINATION_PREFIX + p); err == nil { - destIds := x.([]interface{}) - for _, dId := range destIds { - if dId == mb.DestinationId { - mb.Value += amount - counted = true - break - } + destIds := x.(map[interface{}]struct{}) + if _, found := destIds[mb.DestinationId]; found { + mb.Value += amount + counted = true + break } } if counted { diff --git a/utils/apitpdata.go b/utils/apitpdata.go index d27714cac..26c8dfe08 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -624,14 +624,14 @@ type AttrCacheStats struct { // Add in the future filters here maybe so we avoid } type CacheStats struct { - Destinations int64 - RatingPlans int64 - RatingProfiles int64 - Actions int64 - SharedGroups int64 - RatingAliases int64 - AccountAliases int64 - DerivedChargers int64 + Destinations int + RatingPlans int + RatingProfiles int + Actions int + SharedGroups int + RatingAliases int + AccountAliases int + DerivedChargers int } type AttrCachedItemAge struct { diff --git a/utils/slice.go b/utils/slice.go index bec15c525..964e95d75 100644 --- a/utils/slice.go +++ b/utils/slice.go @@ -32,16 +32,6 @@ func IsSliceMember(ss []string, s string) bool { return false } -// Binary string search in slice -// returns true if found and the index -func GetSliceMemberIndex(ss []string, s string) (int, bool) { - sort.Strings(ss) - if i := sort.SearchStrings(ss, s); i < len(ss) && ss[i] == s { - return i, true - } - return len(ss), false -} - //Iterates over slice members and returns true if one starts with prefix func SliceMemberHasPrefix(ss []string, prfx string) bool { for _, mbr := range ss { @@ -51,11 +41,3 @@ func SliceMemberHasPrefix(ss []string, prfx string) bool { } return false } - -func ConvertInterfaceSliceToStringMap(is []interface{}) (result map[string]int) { - result = make(map[string]int) - for index, i := range is { - result[i.(string)] = index - } - return result -}