optimized prefixes container

apier/v1/tpdestinations.go

@@ -21,6 +21,7 @@ package apier
 import (
 	"errors"
 	"fmt"
+
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
@@ -30,7 +31,7 @@ func (self *ApierV1) SetTPDestination(attrs utils.TPDestination, reply *string)
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationId", "Prefixes"}); len(missing) != 0 { //Params missing
 		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
 	}
-	if err := self.StorDb.SetTPDestination(attrs.TPid, &engine.Destination{attrs.DestinationId, attrs.Prefixes}); err != nil {
+	if err := self.StorDb.SetTPDestination(attrs.TPid, &engine.Destination{Id: attrs.DestinationId, Prefixes: attrs.Prefixes}); err != nil {
 		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
 	}
 	*reply = "OK"

engine/destinations.go

@@ -24,31 +24,67 @@ import (
 	"github.com/cgrates/cgrates/utils"
 )
 
+const (
+	LONG_PREFIX_SLICE_LENGTH = 30
+)
+
 /*
 Structure that gathers multiple destination prefixes under a common id.
 */
 type Destination struct {
-	Id       string
-	Prefixes map[string]interface{}
+	Id              string
+	Prefixes        []string
+	longPrefixesMap map[string]interface{}
 }
 
-func (d *Destination) containsPrefix(prefix string) (precision int) {
+// returns prefix precision
+func (d *Destination) containsPrefix(prefix string) int {
 	if d == nil {
-		return
+		return 0
 	}
-	for i, p := range utils.SplitPrefix(prefix) {
-		if _, found := d.Prefixes[p]; found {
-			return len(prefix) - i
+	if d.Prefixes != nil {
+		for _, p := range d.Prefixes {
+			if strings.Index(prefix, p) == 0 {
+				return len(p)
+			}
 		}
 	}
-	return
+	if d.longPrefixesMap != nil {
+		for i, p := range utils.SplitPrefix(prefix) {
+			if _, found := d.longPrefixesMap[p]; found {
+				return len(prefix) - i
+			}
+		}
+	}
+	return 0
 }
 
 func (d *Destination) String() (result string) {
 	result = d.Id + ": "
-	for k, _ := range d.Prefixes {
-		result += k + ", "
+	if d.Prefixes != nil {
+		for _, k := range d.Prefixes {
+			result += k + ", "
+		}
+	}
+	if d.longPrefixesMap != nil {
+		for k, _ := range d.longPrefixesMap {
+			result += k + ", "
+		}
 	}
 	result = strings.TrimRight(result, ", ")
 	return result
 }
+
+func (d *Destination) AddPrefix(pfx string) {
+	d.Prefixes = append(d.Prefixes, pfx)
+}
+
+func (d *Destination) OptimizePrefixes() {
+	if len(d.Prefixes) > LONG_PREFIX_SLICE_LENGTH {
+		d.longPrefixesMap = make(map[string]interface{})
+		for _, p := range d.Prefixes {
+			d.longPrefixesMap[p] = nil
+		}
+		d.Prefixes = nil
+	}
+}

engine/destinations_test.go

@@ -20,6 +20,7 @@ package engine
 
 import (
 	"encoding/json"
+	"strconv"
 
 	"github.com/cgrates/cgrates/cache2go"
 
@@ -27,7 +28,7 @@ import (
 )
 
 func TestDestinationStoreRestore(t *testing.T) {
-	nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
+	nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}}
 	s, _ := json.Marshal(nationale)
 	d1 := &Destination{Id: "nat"}
 	json.Unmarshal(s, d1)
@@ -38,22 +39,19 @@ func TestDestinationStoreRestore(t *testing.T) {
 }
 
 func TestDestinationStorageStore(t *testing.T) {
-	nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
+	nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}}
 	err := storageGetter.SetDestination(nationale)
 	if err != nil {
 		t.Error("Error storing destination: ", err)
 	}
 	result, err := storageGetter.GetDestination(nationale.Id, false)
-	_, a := nationale.Prefixes["0257"]
-	_, b := nationale.Prefixes["0256"]
-	_, c := nationale.Prefixes["0723"]
-	if !a || !b || !c {
+	if nationale.containsPrefix("0257") == 0 || nationale.containsPrefix("0256") == 0 || nationale.containsPrefix("0723") == 0 {
 		t.Errorf("Expected %q was %q", nationale, result)
 	}
 }
 
 func TestDestinationContainsPrefix(t *testing.T) {
-	nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
+	nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}}
 	precision := nationale.containsPrefix("0256")
 	if precision != len("0256") {
 		t.Error("Should contain prefix: ", nationale)
@@ -61,7 +59,7 @@ func TestDestinationContainsPrefix(t *testing.T) {
 }
 
 func TestDestinationContainsPrefixLong(t *testing.T) {
-	nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
+	nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}}
 	precision := nationale.containsPrefix("0256723045")
 	if precision != len("0256") {
 		t.Error("Should contain prefix: ", nationale)
@@ -69,7 +67,7 @@ func TestDestinationContainsPrefixLong(t *testing.T) {
 }
 
 func TestDestinationContainsPrefixWrong(t *testing.T) {
-	nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
+	nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}}
 	precision := nationale.containsPrefix("01234567")
 	if precision != 0 {
 		t.Error("Should not contain prefix: ", nationale)
@@ -104,59 +102,32 @@ func TestDestinationGetNotExistsCache(t *testing.T) {
 	}
 }
 
-/*
-func TestConcurrentDestReadWrite(t *testing.T) {
-	dst1 := &Destination{Id: "TST_1", Prefixes: []string{"1"}}
-	err := storageGetter.SetDestination(dst1)
-	if err != nil {
-		t.Error("Error setting  destination: ", err)
+func TestDestinationOptimzeShort(t *testing.T) {
+	d := &Destination{}
+	for i := 0; i < LONG_PREFIX_SLICE_LENGTH; i++ {
+		d.AddPrefix(strconv.Itoa(i))
 	}
-	rec := 500
-	go func() {
-		for i := 0; i < rec; i++ {
-			storageGetter.SetDestination(&Destination{Id: fmt.Sprintf("TST_%d", i), Prefixes: []string{"1"}})
-		}
-	}()
-
-	for i := 0; i < rec; i++ {
-		dst2, err := storageGetter.GetDestination(dst1.Id)
-		if err != nil {
-			t.Error("Error retrieving destination: ", err)
-		}
-		if !reflect.DeepEqual(dst1, dst2) {
-			t.Error("Cannot retrieve properly the destination 1", dst1, dst2)
-		}
+	d.OptimizePrefixes()
+	if d.Prefixes == nil || d.longPrefixesMap != nil {
+		t.Logf("Error optimizing destinations %+v", d)
 	}
 }
 
-func TestNonConcurrentDestReadWrite(t *testing.T) {
-	dst1 := &Destination{Id: "TST_1", Prefixes: []string{"1"}}
-	err := storageGetter.SetDestination(dst1)
-	if err != nil {
-		t.Error("Error setting destination: ", err)
+func TestDestinationOptimzeLong(t *testing.T) {
+	d := &Destination{}
+	for i := 0; i < LONG_PREFIX_SLICE_LENGTH+1; i++ {
+		d.AddPrefix(strconv.Itoa(i))
 	}
-	rec := 10000
-	//go func(){
-	for i := 0; i < rec; i++ {
-		storageGetter.SetDestination(&Destination{Id: fmt.Sprintf("TST_%d", i), Prefixes: []string{"1"}})
-	}
-	//}()
-
-	for i := 0; i < rec; i++ {
-		dst2, err := storageGetter.GetDestination(dst1.Id)
-		if err != nil {
-			t.Error("Error retrieving destination: ", err)
-		}
-		if !reflect.DeepEqual(dst1, dst2) {
-			t.Error("Cannot retrieve properly the destination 1", dst1, dst2)
-		}
+	d.OptimizePrefixes()
+	if d.Prefixes != nil || d.longPrefixesMap == nil {
+		t.Logf("Error optimizing destinations %+v", d)
 	}
 }
-*/
+
 /********************************* Benchmarks **********************************/
 
 func BenchmarkDestinationStorageStoreRestore(b *testing.B) {
-	nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
+	nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}}
 	for i := 0; i < b.N; i++ {
 		storageGetter.SetDestination(nationale)
 		storageGetter.GetDestination(nationale.Id, true)

engine/history_test.go

@@ -31,3 +31,20 @@ func TestHistoryRatinPlans(t *testing.T) {
 		t.Error("Error in destination history content:", scribe.RpBuf.String())
 	}
 }
+
+func TestHistoryDestinations(t *testing.T) {
+	scribe := historyScribe.(*history.MockScribe)
+	expected := `[{"Key":"ALL","Object":{"Id":"ALL","Prefixes":["49","41","43"]}}
+{"Key":"GERMANY","Object":{"Id":"GERMANY","Prefixes":["49"]}}
+{"Key":"GERMANY_O2","Object":{"Id":"GERMANY_O2","Prefixes":["41"]}}
+{"Key":"GERMANY_PREMIUM","Object":{"Id":"GERMANY_PREMIUM","Prefixes":["43"]}}
+{"Key":"NAT","Object":{"Id":"NAT","Prefixes":["0256","0257","0723"]}}
+{"Key":"PSTN_70","Object":{"Id":"PSTN_70","Prefixes":["+4970"]}}
+{"Key":"PSTN_71","Object":{"Id":"PSTN_71","Prefixes":["+4971"]}}
+{"Key":"PSTN_72","Object":{"Id":"PSTN_72","Prefixes":["+4972"]}}
+{"Key":"RET","Object":{"Id":"RET","Prefixes":["0723","0724"]}}
+{"Key":"nat","Object":{"Id":"nat","Prefixes":["0257","0256","0723"]}}]`
+	if scribe.DestBuf.String() != expected {
+		t.Error("Error in destination history content:", scribe.DestBuf.String())
+	}
+}

engine/loader_csv.go

@@ -199,10 +199,10 @@ func (csvr *CSVReader) LoadDestinations() (err error) {
 			}
 		}
 		if dest == nil {
-			dest = &Destination{Id: tag, Prefixes: make(map[string]interface{}, 1)}
+			dest = &Destination{Id: tag}
 			csvr.destinations = append(csvr.destinations, dest)
 		}
-		dest.Prefixes[record[1]] = nil
+		dest.AddPrefix(record[1])
 	}
 	return
 }

engine/loader_csv_test.go

@@ -151,39 +151,39 @@ func TestLoadDestinations(t *testing.T) {
 	for _, d := range csvr.destinations {
 		switch d.Id {
 		case "NAT":
-			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`0256`: nil, `0257`: nil, `0723`: nil}) {
+			if !reflect.DeepEqual(d.Prefixes, []string{`0256`, `0257`, `0723`}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "ALL":
-			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`49`: nil, `41`: nil, `43`: nil}) {
+			if !reflect.DeepEqual(d.Prefixes, []string{`49`, `41`, `43`}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "RET":
-			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`0723`: nil, `0724`: nil}) {
+			if !reflect.DeepEqual(d.Prefixes, []string{`0723`, `0724`}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "GERMANY":
-			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`49`: nil}) {
+			if !reflect.DeepEqual(d.Prefixes, []string{`49`}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "GERMANY_O2":
-			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`41`: nil}) {
+			if !reflect.DeepEqual(d.Prefixes, []string{`41`}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "GERMANY_PREMIUM":
-			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`43`: nil}) {
+			if !reflect.DeepEqual(d.Prefixes, []string{`43`}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "PSTN_71":
-			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`+4971`: nil}) {
+			if !reflect.DeepEqual(d.Prefixes, []string{`+4971`}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "PSTN_72":
-			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`+4972`: nil}) {
+			if !reflect.DeepEqual(d.Prefixes, []string{`+4972`}) {
 				t.Error("Faild to load destinations", d)
 			}
 		case "PSTN_70":
-			if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`+4970`: nil}) {
+			if !reflect.DeepEqual(d.Prefixes, []string{`+4970`}) {
 				t.Error("Faild to load destinations", d)
 			}
 		default:

engine/storage_map.go

@@ -158,6 +158,7 @@ func (ms *MapStorage) GetDestination(key string, checkDb bool) (dest *Destinatio
 	if values, ok := ms.dict[key]; ok {
 		dest = &Destination{Id: key}
 		err = ms.ms.Unmarshal(values, dest)
+		dest.OptimizePrefixes()
 		cache2go.Cache(key, dest)
 	} else {
 		return nil, errors.New("not found")

engine/storage_redis.go

@@ -244,6 +244,7 @@ func (rs *RedisStorage) GetDestination(key string, checkDb bool) (dest *Destinat
 		r.Close()
 		dest = new(Destination)
 		err = rs.ms.Unmarshal(out, dest)
+		dest.OptimizePrefixes()
 		cache2go.Cache(key, dest)
 	}
 	return

engine/storage_sql.go

@@ -164,7 +164,7 @@ func (self *SQLStorage) GetTPDestination(tpid, destTag string) (*Destination, er
 		return nil, err
 	}
 	defer rows.Close()
-	d := &Destination{Id: destTag, Prefixes: make(map[string]interface{}, 1)}
+	d := &Destination{Id: destTag}
 	i := 0
 	for rows.Next() {
 		i++ //Keep here a reference so we know we got at least one prefix
@@ -173,7 +173,7 @@ func (self *SQLStorage) GetTPDestination(tpid, destTag string) (*Destination, er
 		if err != nil {
 			return nil, err
 		}
-		d.Prefixes[pref] = nil
+		d.AddPrefix(pref)
 	}
 	if i == 0 {
 		return nil, nil
@@ -804,10 +804,10 @@ func (self *SQLStorage) GetTpDestinations(tpid, tag string) ([]*Destination, err
 			}
 		}
 		if dest == nil {
-			dest = &Destination{Id: tag, Prefixes: make(map[string]interface{}, 1)}
+			dest = &Destination{Id: tag}
 			dests = append(dests, dest)
 		}
-		dest.Prefixes[prefix] = nil
+		dest.AddPrefix(prefix)
 	}
 	return dests, nil
 }

engine/storage_test.go

@@ -96,13 +96,13 @@ func TestStorageDestinationContainsPrefixNotExisting(t *testing.T) {
 }
 
 func TestPreCacheRefresh(t *testing.T) {
-	storageGetter.SetDestination(&Destination{"T11", map[string]interface{}{"0": nil}})
+	storageGetter.SetDestination(&Destination{"T11", []string{"0"}, nil})
 	storageGetter.GetDestination("T11", false)
-	storageGetter.SetDestination(&Destination{"T11", map[string]interface{}{"1": nil}})
+	storageGetter.SetDestination(&Destination{"T11", []string{"1"}, nil})
 	storageGetter.PreCache(nil, nil, nil, nil)
 	d, err := storageGetter.GetDestination("T11", false)
-	_, ok := d.Prefixes["1"]
-	if err != nil || !ok {
+	p := d.containsPrefix("1")
+	if err != nil || p == 0 {
 		t.Error("Error refreshing cache:", d)
 	}
 }

engine/tpimporter_csv.go

@@ -115,7 +115,7 @@ func (self *TPCSVImporter) importDestinations(fn string) error {
 			}
 			continue
 		}
-		dst := &Destination{record[0], map[string]interface{}{record[1]: nil}}
+		dst := &Destination{record[0], []string{record[1]}, nil}
 		if err := self.StorDb.SetTPDestination(self.TPid, dst); err != nil {
 			if self.Verbose {
 				log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error())

engine/userbalance_test.go

@@ -24,8 +24,8 @@ import (
 )
 
 var (
-	NAT = &Destination{Id: "NAT", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}}
-	RET = &Destination{Id: "RET", Prefixes: map[string]interface{}{"0723": nil, "0724": nil}}
+	NAT = &Destination{Id: "NAT", Prefixes: []string{"0257", "0256", "0723"}}
+	RET = &Destination{Id: "RET", Prefixes: []string{"0723", "0724"}}
 )
 
 func init() {

utils/apitpdata.go

@@ -26,9 +26,9 @@ import (
 )
 
 type TPDestination struct {
-	TPid          string                 // Tariff plan id
-	DestinationId string                 // Destination id
-	Prefixes      map[string]interface{} // Prefixes attached to this destination
+	TPid          string   // Tariff plan id
+	DestinationId string   // Destination id
+	Prefixes      []string // Prefixes attached to this destination
 }
 
 // This file deals with tp_* data definition