more optimizations for large destination keys

cmd/stress/cgr-spansstress/cgr-spansstress.go

@@ -55,8 +55,8 @@ func main() {
 		Direction:    "*out",
 		TOR:          "call",
 		Tenant:       "cgrates.org",
-		Subject:      "1001",
-		Destination:  "+49",
+		Subject:      "dan",
+		Destination:  "+31676016500",
 	}
 	getter, err := engine.ConfigureDataStorage(utils.REDIS, "127.0.0.1", "6379", "10", "", "", utils.MSGPACK)
 	if err != nil {
@@ -75,7 +75,7 @@ func main() {
 	j := 0
 	start := time.Now()
 	for i := 0; i < *runs; i++ {
-		result, err = cd.Debit()
+		result, err = cd.GetCost()
 		if *memprofile != "" {
 			runtime.MemProfileRate = 1
 			runtime.GC()

engine/destinations.go

@@ -20,6 +20,8 @@ package engine
 
 import (
 	"strings"
+
+	"github.com/cgrates/cgrates/utils"
 )
 
 /*
@@ -27,16 +29,16 @@ Structure that gathers multiple destination prefixes under a common id.
 */
 type Destination struct {
 	Id       string
-	Prefixes []string
+	Prefixes map[string]interface{}
 }
 
 func (d *Destination) containsPrefix(prefix string) (precision int) {
 	if d == nil {
 		return
 	}
-	for _, p := range d.Prefixes {
-		if strings.Index(prefix, p) == 0 && len(p) > precision {
-			precision = len(p)
+	for i, p := range utils.SplitPrefix(prefix) {
+		if _, found := d.Prefixes[p]; found {
+			return len(prefix) - i
 		}
 	}
 	return
@@ -44,8 +46,8 @@ func (d *Destination) containsPrefix(prefix string) (precision int) {
 
 func (d *Destination) String() (result string) {
 	result = d.Id + ": "
-	for _, p := range d.Prefixes {
-		result += p + ", "
+	for k, _ := range d.Prefixes {
+		result += k + ", "
 	}
 	result = strings.TrimRight(result, ", ")
 	return result

engine/destinations_test.go

@@ -22,13 +22,12 @@ import (
 	"encoding/json"
 
 	"github.com/cgrates/cgrates/cache2go"
-	"github.com/cgrates/cgrates/utils"
 
 	"testing"
 )
 
 func TestDestinationStoreRestore(t *testing.T) {
-	nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}}
+	nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
 	s, _ := json.Marshal(nationale)
 	d1 := &Destination{Id: "nat"}
 	json.Unmarshal(s, d1)
@@ -39,20 +38,22 @@ func TestDestinationStoreRestore(t *testing.T) {
 }
 
 func TestDestinationStorageStore(t *testing.T) {
-	nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}}
+	nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
 	err := storageGetter.SetDestination(nationale)
 	if err != nil {
 		t.Error("Error storing destination: ", err)
 	}
 	result, err := storageGetter.GetDestination(nationale.Id, false)
-	var ss utils.StringSlice = result.Prefixes
-	if !ss.Contains("0257") || !ss.Contains("0256") || !ss.Contains("0723") {
+	_, a := nationale.Prefixes["0257"]
+	_, b := nationale.Prefixes["0256"]
+	_, c := nationale.Prefixes["0723"]
+	if !a || !b || !c {
 		t.Errorf("Expected %q was %q", nationale, result)
 	}
 }
 
 func TestDestinationContainsPrefix(t *testing.T) {
-	nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}}
+	nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
 	precision := nationale.containsPrefix("0256")
 	if precision != len("0256") {
 		t.Error("Should contain prefix: ", nationale)
@@ -60,7 +61,7 @@ func TestDestinationContainsPrefix(t *testing.T) {
 }
 
 func TestDestinationContainsPrefixLong(t *testing.T) {
-	nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}}
+	nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
 	precision := nationale.containsPrefix("0256723045")
 	if precision != len("0256") {
 		t.Error("Should contain prefix: ", nationale)
@@ -68,7 +69,7 @@ func TestDestinationContainsPrefixLong(t *testing.T) {
 }
 
 func TestDestinationContainsPrefixWrong(t *testing.T) {
-	nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}}
+	nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
 	precision := nationale.containsPrefix("01234567")
 	if precision != 0 {
 		t.Error("Should not contain prefix: ", nationale)
@@ -155,7 +156,7 @@ func TestNonConcurrentDestReadWrite(t *testing.T) {
 /********************************* Benchmarks **********************************/
 
 func BenchmarkDestinationStorageStoreRestore(b *testing.B) {
-	nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}}
+	nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
 	for i := 0; i < b.N; i++ {
 		storageGetter.SetDestination(nationale)
 		storageGetter.GetDestination(nationale.Id, true)

engine/history_test.go

@@ -19,23 +19,15 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package engine
 
 import (
-	"github.com/cgrates/cgrates/history"
+	"strings"
 	"testing"
+
+	"github.com/cgrates/cgrates/history"
 )
 
-func TestHistoryDestinations(t *testing.T) {
+func TestHistoryRatinPlans(t *testing.T) {
 	scribe := historyScribe.(*history.MockScribe)
-	expected := `[{"Key":"ALL","Object":{"Id":"ALL","Prefixes":["49","41","43"]}}
-{"Key":"GERMANY","Object":{"Id":"GERMANY","Prefixes":["49"]}}
-{"Key":"GERMANY_O2","Object":{"Id":"GERMANY_O2","Prefixes":["41"]}}
-{"Key":"GERMANY_PREMIUM","Object":{"Id":"GERMANY_PREMIUM","Prefixes":["43"]}}
-{"Key":"NAT","Object":{"Id":"NAT","Prefixes":["0256","0257","0723"]}}
-{"Key":"PSTN_70","Object":{"Id":"PSTN_70","Prefixes":["+4970"]}}
-{"Key":"PSTN_71","Object":{"Id":"PSTN_71","Prefixes":["+4971"]}}
-{"Key":"PSTN_72","Object":{"Id":"PSTN_72","Prefixes":["+4972"]}}
-{"Key":"RET","Object":{"Id":"RET","Prefixes":["0723","0724"]}}
-{"Key":"nat","Object":{"Id":"nat","Prefixes":["0257","0256","0723"]}}]`
-	if scribe.DestBuf.String() != expected {
-		t.Error("Error in destination history content:", scribe.DestBuf.String())
+	if !strings.Contains(scribe.RpBuf.String(), `{"Key":"*out:vdf:0:minu","Object":{"Id":"*out:vdf:0:minu","RatingPlanActivations":[{"ActivationTime":"2012-01-01T00:00:00Z","RatingPlanId":"EVENING","FallbackKeys":null}]}}`) {
+		t.Error("Error in destination history content:", scribe.RpBuf.String())
 	}
 }

engine/loader_csv.go

@@ -22,11 +22,12 @@ import (
 	"encoding/csv"
 	"errors"
 	"fmt"
-	"github.com/cgrates/cgrates/utils"
 	"log"
 	"os"
 	"strconv"
 	"strings"
+
+	"github.com/cgrates/cgrates/utils"
 )
 
 type CSVReader struct {
@@ -198,10 +199,10 @@ func (csvr *CSVReader) LoadDestinations() (err error) {
 			}
 		}
 		if dest == nil {
-			dest = &Destination{Id: tag}
+			dest = &Destination{Id: tag, Prefixes: make(map[string]interface{}, 1)}
 			csvr.destinations = append(csvr.destinations, dest)
 		}
-		dest.Prefixes = append(dest.Prefixes, record[1])
+		dest.Prefixes[record[1]] = nil
 	}
 	return
 }

engine/loader_csv_test.go

@@ -151,39 +151,39 @@ func TestLoadDestinations(t *testing.T) {
 	for _, d := range csvr.destinations {
 		switch d.Id {
 		case "NAT":
-			if !reflect.DeepEqual(d.Prefixes, []string{`0256`, `0257`, `0723`}) {
+			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`0256`: nil, `0257`: nil, `0723`: nil}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "ALL":
-			if !reflect.DeepEqual(d.Prefixes, []string{`49`, `41`, `43`}) {
+			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`49`: nil, `41`: nil, `43`: nil}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "RET":
-			if !reflect.DeepEqual(d.Prefixes, []string{`0723`, `0724`}) {
+			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`0723`: nil, `0724`: nil}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "GERMANY":
-			if !reflect.DeepEqual(d.Prefixes, []string{`49`}) {
+			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`49`: nil}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "GERMANY_O2":
-			if !reflect.DeepEqual(d.Prefixes, []string{`41`}) {
+			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`41`: nil}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "GERMANY_PREMIUM":
-			if !reflect.DeepEqual(d.Prefixes, []string{`43`}) {
+			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`43`: nil}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "PSTN_71":
-			if !reflect.DeepEqual(d.Prefixes, []string{`+4971`}) {
+			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`+4971`: nil}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "PSTN_72":
-			if !reflect.DeepEqual(d.Prefixes, []string{`+4972`}) {
+			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`+4972`: nil}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "PSTN_70":
-			if !reflect.DeepEqual(d.Prefixes, []string{`+4970`}) {
+			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`+4970`: nil}) {
 				t.Error("Faild to load destinations", d)
 			}
 		default:
@@ -554,7 +554,7 @@ func TestLoadRatingProfiles(t *testing.T) {
 		RatingPlanActivations: RatingPlanActivations{&RatingPlanActivation{
 			ActivationTime: time.Date(2013, 10, 1, 0, 0, 0, 0, time.UTC),
 			RatingPlanId:   "TDRT",
-			FallbackKeys:   []string{"*out:test:0:rif", "*out:test:0:danb"},
+			FallbackKeys:   []string{"*out:test:0:danb", "*out:test:0:rif"},
 		}},
 	}
 	if !reflect.DeepEqual(rp, expected) {

engine/storage_map.go

@@ -46,6 +46,9 @@ func (ms *MapStorage) Flush() error {
 }
 
 func (ms *MapStorage) PreCache(dKeys, rpKeys, rpfKeys, actKeys []string) error {
+	if dKeys == nil && rpKeys == nil && rpfKeys == nil && actKeys == nil {
+		cache2go.Flush()
+	}
 	for k, _ := range ms.dict {
 		if strings.HasPrefix(k, DESTINATION_PREFIX) {
 			cache2go.RemKey(k)

engine/storage_redis.go

@@ -69,6 +69,9 @@ func (rs *RedisStorage) Flush() (err error) {
 }
 
 func (rs *RedisStorage) PreCache(dKeys, rpKeys, rpfKeys, actKeys []string) (err error) {
+	if dKeys == nil && rpKeys == nil && rpfKeys == nil && actKeys == nil {
+		cache2go.Flush()
+	}
 	if dKeys == nil {
 		Logger.Info("Caching all destinations")
 		if dKeys, err = rs.db.Keys(DESTINATION_PREFIX + "*"); err != nil {

engine/storage_sql.go

@@ -23,10 +23,11 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
-	"github.com/cgrates/cgrates/utils"
 	"io/ioutil"
 	"strings"
 	"time"
+
+	"github.com/cgrates/cgrates/utils"
 )
 
 type SQLStorage struct {
@@ -163,7 +164,7 @@ func (self *SQLStorage) GetTPDestination(tpid, destTag string) (*Destination, er
 		return nil, err
 	}
 	defer rows.Close()
-	d := &Destination{Id: destTag}
+	d := &Destination{Id: destTag, Prefixes: make(map[string]interface{}, 1)}
 	i := 0
 	for rows.Next() {
 		i++ //Keep here a reference so we know we got at least one prefix
@@ -172,7 +173,7 @@ func (self *SQLStorage) GetTPDestination(tpid, destTag string) (*Destination, er
 		if err != nil {
 			return nil, err
 		}
-		d.Prefixes = append(d.Prefixes, pref)
+		d.Prefixes[pref] = nil
 	}
 	if i == 0 {
 		return nil, nil
@@ -186,11 +187,13 @@ func (self *SQLStorage) SetTPDestination(tpid string, dest *Destination) error {
 	}
 	var buffer bytes.Buffer // Use bytes buffer istead of string concatenation since that becomes quite heavy on large prefixes
 	buffer.WriteString(fmt.Sprintf("INSERT INTO %s (tpid, tag, prefix) VALUES ", utils.TBL_TP_DESTINATIONS))
-	for idx, prefix := range dest.Prefixes {
+	idx := 0
+	for prefix, _ := range dest.Prefixes {
 		if idx != 0 {
 			buffer.WriteRune(',')
 		}
 		buffer.WriteString(fmt.Sprintf("('%s','%s','%s')", tpid, dest.Id, prefix))
+		idx++
 	}
 	buffer.WriteString(" ON DUPLICATE KEY UPDATE prefix=values(prefix)")
 	if _, err := self.Db.Exec(buffer.String()); err != nil {
@@ -801,10 +804,10 @@ func (self *SQLStorage) GetTpDestinations(tpid, tag string) ([]*Destination, err
 			}
 		}
 		if dest == nil {
-			dest = &Destination{Id: tag}
+			dest = &Destination{Id: tag, Prefixes: make(map[string]interface{}, 1)}
 			dests = append(dests, dest)
 		}
-		dest.Prefixes = append(dest.Prefixes, prefix)
+		dest.Prefixes[prefix] = nil
 	}
 	return dests, nil
 }

engine/storage_test.go

@@ -96,11 +96,13 @@ func TestStorageDestinationContainsPrefixNotExisting(t *testing.T) {
 }
 
 func TestPreCacheRefresh(t *testing.T) {
-	storageGetter.SetDestination(&Destination{"T11", []string{"0"}})
+	storageGetter.SetDestination(&Destination{"T11", map[string]interface{}{"0": nil}})
 	storageGetter.GetDestination("T11", false)
-	storageGetter.SetDestination(&Destination{"T11", []string{"1"}})
+	storageGetter.SetDestination(&Destination{"T11", map[string]interface{}{"1": nil}})
 	storageGetter.PreCache(nil, nil, nil, nil)
-	if d, err := storageGetter.GetDestination("T11", false); err != nil || d.Prefixes[0] != "1" {
+	d, err := storageGetter.GetDestination("T11", false)
+	_, ok := d.Prefixes["1"]
+	if err != nil || !ok {
 		t.Error("Error refreshing cache:", d)
 	}
 }

engine/tpimporter_csv.go

@@ -19,11 +19,12 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package engine
 
 import (
-	"github.com/cgrates/cgrates/utils"
 	"io"
 	"io/ioutil"
 	"log"
 	"strconv"
+
+	"github.com/cgrates/cgrates/utils"
 )
 
 // Import tariff plan from csv into storDb
@@ -114,7 +115,7 @@ func (self *TPCSVImporter) importDestinations(fn string) error {
 			}
 			continue
 		}
-		dst := &Destination{record[0], []string{record[1]}}
+		dst := &Destination{record[0], map[string]interface{}{record[1]: nil}}
 		if err := self.StorDb.SetTPDestination(self.TPid, dst); err != nil {
 			if self.Verbose {
 				log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error())

engine/userbalance_test.go

@@ -24,8 +24,8 @@ import (
 )
 
 var (
-	NAT = &Destination{Id: "NAT", Prefixes: []string{"0257", "0256", "0723"}}
-	RET = &Destination{Id: "RET", Prefixes: []string{"0723", "0724"}}
+	NAT = &Destination{Id: "NAT", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
+	RET = &Destination{Id: "RET", Prefixes: map[string]interface{}{"0723": nil, "0724": nil}}
 )
 
 func init() {

utils/apitpdata.go

@@ -20,14 +20,15 @@ package utils
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 )
 
 type TPDestination struct {
-	TPid          string   // Tariff plan id
-	DestinationId string   // Destination id
-	Prefixes      []string // Prefixes attached to this destination
+	TPid          string                 // Tariff plan id
+	DestinationId string                 // Destination id
+	Prefixes      map[string]interface{} // Prefixes attached to this destination
 }
 
 // This file deals with tp_* data definition
@@ -171,17 +172,25 @@ type TPRatingActivation struct {
 
 // Helper to return the subject fallback keys we need in dataDb
 func FallbackSubjKeys(direction, tenant, tor, fallbackSubjects string) []string {
-	var subjKeys []string
+	var sslice sort.StringSlice
 	if len(fallbackSubjects) != 0 {
-		var sslice StringSlice
 		for _, fbs := range strings.Split(fallbackSubjects, string(FALLBACK_SEP)) {
 			newKey := fmt.Sprintf("%s:%s:%s:%s", direction, tenant, tor, fbs)
-			if !sslice.Contains(newKey) {
-				subjKeys = append(subjKeys, newKey)
-			}
+			i := sslice.Search(newKey)
+			if i < len(sslice) && sslice[i] != newKey {
+				// not found so insert it
+				sslice = append(sslice, "")
+				copy(sslice[i+1:], sslice[i:])
+				sslice[i] = newKey
+			} else {
+				if i == len(sslice) {
+					// not found and at the end
+					sslice = append(sslice, newKey)
+				}
+			} // newKey was foundfound
 		}
 	}
-	return subjKeys
+	return sslice
 }
 
 type AttrTPRatingProfileIds struct {

utils/coreutils.go

@@ -132,22 +132,12 @@ func RoundTo(whole, amount time.Duration) time.Duration {
 	return time.Duration((w - math.Mod(a, w)) + a)
 }
 
-type StringSlice []string
-
-func (ss StringSlice) Contains(needle string) bool {
-	for _, hay := range ss {
-		if hay == needle {
-			return true
-		}
-	}
-	return false
-}
-
-func SplitPrefixInterface(prefix string) []interface{} {
-	var subs []interface{}
+func SplitPrefix(prefix string) []string {
+	length := int(math.Max(float64(len(prefix)-1), 0))
+	subs := make([]string, length)
 	max := len(prefix)
-	for i := 0; i < len(prefix)-1; i++ {
-		subs = append(subs, prefix[:max-i])
+	for i := 0; i < length; i++ {
+		subs[i] = prefix[:max-i]
 	}
 	return subs
 }

utils/utils_test.go

@@ -235,14 +235,14 @@ func TestRound(t *testing.T) {
 }
 
 func TestSplitPrefix(t *testing.T) {
-	a := SplitPrefixInterface("0123456789")
+	a := SplitPrefix("0123456789")
 	if len(a) != 9 {
 		t.Error("Error splitting prefix: ", a)
 	}
 }
 
 func TestSplitPrefixEmpty(t *testing.T) {
-	a := SplitPrefixInterface("")
+	a := SplitPrefix("")
 	if len(a) != 0 {
 		t.Error("Error splitting prefix: ", a)
 	}