Merge branch 'master' of https://github.com/cgrates/cgrates

apier/v1/apier.go

@@ -22,7 +22,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"log"
 	"path"
 
 	"github.com/cgrates/cgrates/cache2go"
@@ -793,11 +792,9 @@ func (self *ApierV1) ReloadCache(attrs utils.ApiReloadCache, reply *string) erro
 			dcsKeys[idx] = engine.DERIVEDCHARGERS_PREFIX + dc
 		}
 	}
-	log.Print("Cache Rating")
 	if err := self.RatingDb.CacheRating(dstKeys, rpKeys, rpfKeys, rpAlsKeys, lcrKeys); err != nil {
 		return err
 	}
-	log.Print("Cache Accounting")
 	if err := self.AccountDb.CacheAccounting(actKeys, shgKeys, accAlsKeys, dcsKeys); err != nil {
 		return err
 	}

cache2go/cache.go

@@ -158,7 +158,7 @@ func Flush() {
 	}
 }
 
-func CountEntries(prefix string) (result int64) {
+func CountEntries(prefix string) (result int) {
 	mux.RLock()
 	defer mux.RUnlock()
 	return cache.CountEntriesForPrefix(prefix)

cache2go/cache_test.go

@@ -89,7 +89,7 @@ func TestCachePush(t *testing.T) {
 	CachePush("ccc_t1", "1")
 	CachePush("ccc_t1", "2")
 	v, err := GetCached("ccc_t1")
-	if err != nil || len(v.([]interface{})) != 2 {
+	if err != nil || len(v.(map[interface{}]struct{})) != 2 {
 		t.Error("Error in cache push: ", v)
 	}
 }

cache2go/store.go

@@ -16,7 +16,7 @@ type cacheStore interface {
 	GetAge(string) (time.Duration, error)
 	Delete(string)
 	DeletePrefix(string)
-	CountEntriesForPrefix(string) int64
+	CountEntriesForPrefix(string) int
 	GetAllForPrefix(string) (map[string]timestampedValue, error)
 	GetKeysForPrefix(string) []string
 }
@@ -37,21 +37,15 @@ func (cs cacheDoubleStore) Put(key string, value interface{}) {
 }
 
 func (cs cacheDoubleStore) Append(key string, value interface{}) {
-	var elements []interface{}
-	v, err := cs.Get(key)
-	if err == nil {
-		elements = v.([]interface{})
+	var elements map[interface{}]struct{} // using map for faster check if element is present
+	if v, err := cs.Get(key); err == nil {
+		elements = v.(map[interface{}]struct{})
+	} else {
+		elements = make(map[interface{}]struct{})
 	}
 	// check if the val is already present
-	found := false
-	for _, v := range elements {
-		if value == v {
-			found = true
-			break
-		}
-	}
-	if !found {
-		elements = append(elements, value)
+	if _, found := elements[value]; !found {
+		elements[value] = struct{}{}
 	}
 	cache.Put(key, elements)
 }
@@ -87,9 +81,9 @@ func (cs cacheDoubleStore) DeletePrefix(prefix string) {
 	delete(cs, prefix)
 }
 
-func (cs cacheDoubleStore) CountEntriesForPrefix(prefix string) int64 {
+func (cs cacheDoubleStore) CountEntriesForPrefix(prefix string) int {
 	if _, ok := cs[prefix]; ok {
-		return int64(len(cs[prefix]))
+		return len(cs[prefix])
 	}
 	return 0
 }
@@ -116,13 +110,13 @@ func (cs cacheDoubleStore) GetKeysForPrefix(prefix string) (keys []string) {
 // faster to access
 type cacheSimpleStore struct {
 	cache    map[string]timestampedValue
-	counters map[string]int64
+	counters map[string]int
 }
 
 func newSimpleStore() cacheSimpleStore {
 	return cacheSimpleStore{
 		cache:    make(map[string]timestampedValue),
-		counters: make(map[string]int64),
+		counters: make(map[string]int),
 	}
 }
 
@@ -135,22 +129,17 @@ func (cs cacheSimpleStore) Put(key string, value interface{}) {
 }
 
 func (cs cacheSimpleStore) Append(key string, value interface{}) {
-	var elements []interface{}
-	if ti, exists := cs.cache[key]; exists {
-		elements = ti.value.([]interface{})
+	var elements map[interface{}]struct{}
+	if v, err := cs.Get(key); err == nil {
+		elements = v.(map[interface{}]struct{})
+	} else {
+		elements = make(map[interface{}]struct{})
 	}
 	// check if the val is already present
-	found := false
-	for _, v := range elements {
-		if value == v {
-			found = true
-			break
-		}
+	if _, found := elements[value]; !found {
+		elements[value] = struct{}{}
 	}
-	if !found {
-		elements = append(elements, value)
-	}
-	cs.Put(key, elements)
+	cache.Put(key, elements)
 }
 
 func (cs cacheSimpleStore) Get(key string) (interface{}, error) {
@@ -209,7 +198,7 @@ func (cs cacheSimpleStore) descount(key string) {
 	}
 }
 
-func (cs cacheSimpleStore) CountEntriesForPrefix(prefix string) int64 {
+func (cs cacheSimpleStore) CountEntriesForPrefix(prefix string) int {
 	if _, ok := cs.counters[prefix]; ok {
 		return cs.counters[prefix]
 	}

cmd/cgr-engine/cgr-engine.go

@@ -22,10 +22,8 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
-	"runtime"
 	//"runtime"
 	"strconv"
 	"time"
@@ -304,22 +302,7 @@ func writePid() {
 	}
 }
 
-func start() {
-	defer func() {
-		if r := recover(); r != nil {
-			engine.Logger.Crit(fmt.Sprintf("CRITICAL ERROR: %v", r))
-			var stack [8192]byte
-			runtime.Stack(stack[:], false)
-			if tmpFile, err := ioutil.TempFile(os.TempDir(), "cgr_coredump"); err != nil {
-				engine.Logger.Crit(fmt.Sprintf("Cannot create coredump file: %v", err))
-				engine.Logger.Crit(string(stack[:]))
-			} else {
-				tmpFile.Write(stack[:])
-				tmpFile.Close()
-				engine.Logger.Crit(fmt.Sprintf("Core dumped: %s", tmpFile.Name()))
-			}
-		}
-	}()
+func main() {
 	flag.Parse()
 	if *version {
 		fmt.Println("CGRateS " + utils.VERSION)
@@ -529,12 +512,3 @@ func start() {
 	}
 	engine.Logger.Info("Stopped all components. CGRateS shutdown!")
 }
-
-func main() {
-	defer func() {
-		if e := recover(); e != nil {
-			fmt.Fprintf(os.Stderr, "PANIC: %v\n", e)
-		}
-	}()
-	start()
-}

console/status.go

@@ -50,7 +50,7 @@ func (self *CmdStatus) RpcParams() interface{} {
 }
 
 func (self *CmdStatus) RpcResult() interface{} {
-	var s string
+	var s map[string]interface{}
 	return &s
 }
 

data/scripts/pkg/debian/cgrates.init

@@ -23,6 +23,7 @@ GROUP=cgrates
 ENGINE_OPTS=""
 RUNDIR=/var/run/$NAME
 PIDFILE=$RUNDIR/cgr-engine.pid
+COREDUMP=$RUNDIR/cgr-coredump.log
 SCRIPTNAME=/etc/init.d/$NAME
 DEFAULTS=/etc/default/$NAME
 ENABLE=false
@@ -62,7 +63,8 @@ do_start()
 	#   0 if daemon has been started
 	#   1 if daemon was already running
 	#   2 if daemon could not be started
-	start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --test > /dev/null \
+    echo "Started at:" `date`>>$COREDUMP
+	start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --test > /dev/null 2>>$COREDUMP \
                 || return 1
         start-stop-daemon --start --quiet --pidfile $PIDFILE --make-pidfile --exec $DAEMON --chuid $USER:$GROUP --background -- \
                 $ENGINE_OPTS \

engine/account.go

@@ -160,8 +160,8 @@ func (ub *Account) getBalancesForPrefix(prefix, category string, balances Balanc
 		if b.DestinationId != "" && b.DestinationId != utils.ANY {
 			for _, p := range utils.SplitPrefix(prefix, MIN_PREFIX_MATCH) {
 				if x, err := cache2go.GetCached(DESTINATION_PREFIX + p); err == nil {
-					destIds := x.([]interface{})
-					for _, dId := range destIds {
+					destIds := x.(map[interface{}]struct{})
+					for dId, _ := range destIds {
 						if dId == b.DestinationId {
 							b.precision = len(p)
 							usefulBalances = append(usefulBalances, b)

engine/destinations.go

@@ -23,7 +23,6 @@ import (
 	"strings"
 
 	"github.com/cgrates/cgrates/cache2go"
-	"github.com/cgrates/cgrates/utils"
 
 	"github.com/cgrates/cgrates/history"
 )
@@ -75,11 +74,8 @@ func (d *Destination) GetHistoryRecord() history.Record {
 // Reverse search in cache to see if prefix belongs to destination id
 func CachedDestHasPrefix(destId, prefix string) bool {
 	if cached, err := cache2go.GetCached(DESTINATION_PREFIX + prefix); err == nil {
-		for _, cachedDstId := range cached.([]interface{}) {
-			if destId == cachedDstId {
-				return true
-			}
-		}
+		_, found := cached.(map[interface{}]struct{})[destId]
+		return found
 	}
 	return false
 }
@@ -90,17 +86,16 @@ func CleanStalePrefixes(destIds []string) {
 		return
 	}
 	for prefix, idIDs := range prefixMap {
-		dIDs := idIDs.Value().([]interface{})
-		strdIDs := utils.ConvertInterfaceSliceToStringMap(dIDs)
+		dIDs := idIDs.Value().(map[interface{}]struct{})
 		changed := false
 		for _, searchedDID := range destIds {
-			if i, found := strdIDs[searchedDID]; found {
+			if _, found := dIDs[searchedDID]; found {
 				if len(dIDs) == 1 {
 					// remove de prefix from cache
 					cache2go.RemKey(DESTINATION_PREFIX + prefix)
 				} else {
 					// delete the destination from list and put the new list in chache
-					dIDs[i], dIDs = dIDs[len(dIDs)-1], dIDs[:len(dIDs)-1]
+					delete(dIDs, searchedDID)
 					changed = true
 				}
 			}

engine/destinations_test.go

@@ -126,17 +126,18 @@ func TestNonCachedDestWrongPrefix(t *testing.T) {
 }
 
 func TestCleanStalePrefixes(t *testing.T) {
-	cache2go.Cache(DESTINATION_PREFIX+"1", []interface{}{"D1", "D2"})
-	cache2go.Cache(DESTINATION_PREFIX+"2", []interface{}{"D1"})
-	cache2go.Cache(DESTINATION_PREFIX+"3", []interface{}{"D2"})
+	x := struct{}{}
+	cache2go.Cache(DESTINATION_PREFIX+"1", map[interface{}]struct{}{"D1": x, "D2": x})
+	cache2go.Cache(DESTINATION_PREFIX+"2", map[interface{}]struct{}{"D1": x})
+	cache2go.Cache(DESTINATION_PREFIX+"3", map[interface{}]struct{}{"D2": x})
 	CleanStalePrefixes([]string{"D1"})
-	if r, err := cache2go.GetCached(DESTINATION_PREFIX + "1"); err != nil || len(r.([]interface{})) != 1 {
+	if r, err := cache2go.GetCached(DESTINATION_PREFIX + "1"); err != nil || len(r.(map[interface{}]struct{})) != 1 {
 		t.Error("Error cleaning stale destination ids", r)
 	}
 	if r, err := cache2go.GetCached(DESTINATION_PREFIX + "2"); err == nil {
 		t.Error("Error removing stale prefix: ", r)
 	}
-	if r, err := cache2go.GetCached(DESTINATION_PREFIX + "3"); err != nil || len(r.([]interface{})) != 1 {
+	if r, err := cache2go.GetCached(DESTINATION_PREFIX + "3"); err != nil || len(r.(map[interface{}]struct{})) != 1 {
 		t.Error("Error performing stale cleaning: ", r)
 	}
 }

engine/ratingprofile.go

@@ -127,8 +127,8 @@ func (rp *RatingProfile) GetRatingPlansForPrefix(cd *CallDescriptor) (err error)
 		} else {
 			for _, p := range utils.SplitPrefix(cd.Destination, MIN_PREFIX_MATCH) {
 				if x, err := cache2go.GetCached(DESTINATION_PREFIX + p); err == nil {
-					destIds := x.([]interface{})
-					for _, idId := range destIds {
+					destIds := x.(map[interface{}]struct{})
+					for idId, _ := range destIds {
 						dId := idId.(string)
 						if _, ok := rpl.DestinationRates[dId]; ok {
 							rps = rpl.RateIntervalList(dId)

engine/storage_interface.go

@@ -57,6 +57,8 @@ const (
 	CREATE_CDRS_TABLES_SQL       = "create_cdrs_tables.sql"
 	CREATE_TARIFFPLAN_TABLES_SQL = "create_tariffplan_tables.sql"
 	TEST_SQL                     = "TEST_SQL"
+
+	DESTINATIONS_LOAD_THRESHOLD = 0.1
 )
 
 type Storage interface {

engine/storage_map.go

@@ -58,7 +58,7 @@ func (ms *MapStorage) GetKeysForPrefix(string) ([]string, error) {
 
 func (ms *MapStorage) CacheRating(dKeys, rpKeys, rpfKeys, alsKeys, lcrKeys []string) error {
 	cache2go.BeginTransaction()
-	if dKeys == nil {
+	if dKeys == nil || (float64(cache2go.CountEntries(DESTINATION_PREFIX))*DESTINATIONS_LOAD_THRESHOLD < float64(len(dKeys))) {
 		cache2go.RemPrefixKey(DESTINATION_PREFIX)
 	} else {
 		CleanStalePrefixes(dKeys)

engine/storage_redis.go

@@ -73,7 +73,8 @@ func (rs *RedisStorage) GetKeysForPrefix(prefix string) ([]string, error) {
 
 func (rs *RedisStorage) CacheRating(dKeys, rpKeys, rpfKeys, alsKeys, lcrKeys []string) (err error) {
 	cache2go.BeginTransaction()
-	if dKeys == nil {
+	if dKeys == nil || (float64(cache2go.CountEntries(DESTINATION_PREFIX))*DESTINATIONS_LOAD_THRESHOLD < float64(len(dKeys))) {
+		// if need to load more than a half of exiting keys load them all
 		Logger.Info("Caching all destinations")
 		if dKeys, err = rs.db.Keys(DESTINATION_PREFIX + "*"); err != nil {
 			cache2go.RollbackTransaction()

engine/units_counter.go

@@ -78,13 +78,11 @@ func (uc *UnitsCounter) addUnits(amount float64, prefix string) {
 			}
 			for _, p := range utils.SplitPrefix(prefix, MIN_PREFIX_MATCH) {
 				if x, err := cache2go.GetCached(DESTINATION_PREFIX + p); err == nil {
-					destIds := x.([]interface{})
-					for _, dId := range destIds {
-						if dId == mb.DestinationId {
-							mb.Value += amount
-							counted = true
-							break
-						}
+					destIds := x.(map[interface{}]struct{})
+					if _, found := destIds[mb.DestinationId]; found {
+						mb.Value += amount
+						counted = true
+						break
 					}
 				}
 				if counted {

utils/apitpdata.go

@@ -624,14 +624,14 @@ type AttrCacheStats struct { // Add in the future filters here maybe so we avoid
 }
 
 type CacheStats struct {
-	Destinations    int64
-	RatingPlans     int64
-	RatingProfiles  int64
-	Actions         int64
-	SharedGroups    int64
-	RatingAliases   int64
-	AccountAliases  int64
-	DerivedChargers int64
+	Destinations    int
+	RatingPlans     int
+	RatingProfiles  int
+	Actions         int
+	SharedGroups    int
+	RatingAliases   int
+	AccountAliases  int
+	DerivedChargers int
 }
 
 type AttrCachedItemAge struct {

utils/slice.go

@@ -32,16 +32,6 @@ func IsSliceMember(ss []string, s string) bool {
 	return false
 }
 
-// Binary string search in slice
-// returns true if found and the index
-func GetSliceMemberIndex(ss []string, s string) (int, bool) {
-	sort.Strings(ss)
-	if i := sort.SearchStrings(ss, s); i < len(ss) && ss[i] == s {
-		return i, true
-	}
-	return len(ss), false
-}
-
 //Iterates over slice members and returns true if one starts with prefix
 func SliceMemberHasPrefix(ss []string, prfx string) bool {
 	for _, mbr := range ss {
@@ -51,11 +41,3 @@ func SliceMemberHasPrefix(ss []string, prfx string) bool {
 	}
 	return false
 }
-
-func ConvertInterfaceSliceToStringMap(is []interface{}) (result map[string]int) {
-	result = make(map[string]int)
-	for index, i := range is {
-		result[i.(string)] = index
-	}
-	return result
-}