From dbe8bde7bf3f99e5e06f7ac0c437a64fbd08997a Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 16 Dec 2013 19:09:56 +0200 Subject: [PATCH] optimized prefixes container --- apier/v1/tpdestinations.go | 3 +- engine/destinations.go | 56 ++++++++++++++++++++++----- engine/destinations_test.go | 75 ++++++++++++------------------------- engine/history_test.go | 17 +++++++++ engine/loader_csv.go | 4 +- engine/loader_csv_test.go | 18 ++++----- engine/storage_map.go | 1 + engine/storage_redis.go | 1 + engine/storage_sql.go | 8 ++-- engine/storage_test.go | 8 ++-- engine/tpimporter_csv.go | 2 +- engine/userbalance_test.go | 4 +- utils/apitpdata.go | 6 +-- 13 files changed, 115 insertions(+), 88 deletions(-) diff --git a/apier/v1/tpdestinations.go b/apier/v1/tpdestinations.go index 14a27181f..4cd03a36d 100644 --- a/apier/v1/tpdestinations.go +++ b/apier/v1/tpdestinations.go @@ -21,6 +21,7 @@ package apier import ( "errors" "fmt" + "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" ) @@ -30,7 +31,7 @@ func (self *ApierV1) SetTPDestination(attrs utils.TPDestination, reply *string) if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationId", "Prefixes"}); len(missing) != 0 { //Params missing return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing) } - if err := self.StorDb.SetTPDestination(attrs.TPid, &engine.Destination{attrs.DestinationId, attrs.Prefixes}); err != nil { + if err := self.StorDb.SetTPDestination(attrs.TPid, &engine.Destination{Id: attrs.DestinationId, Prefixes: attrs.Prefixes}); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } *reply = "OK" diff --git a/engine/destinations.go b/engine/destinations.go index 55d50d298..68941e28b 100644 --- a/engine/destinations.go +++ b/engine/destinations.go @@ -24,31 +24,67 @@ import ( "github.com/cgrates/cgrates/utils" ) +const ( + LONG_PREFIX_SLICE_LENGTH = 30 +) + /* Structure that gathers multiple destination prefixes under a common id. */ type Destination struct { - Id string - Prefixes map[string]interface{} + Id string + Prefixes []string + longPrefixesMap map[string]interface{} } -func (d *Destination) containsPrefix(prefix string) (precision int) { +// returns prefix precision +func (d *Destination) containsPrefix(prefix string) int { if d == nil { - return + return 0 } - for i, p := range utils.SplitPrefix(prefix) { - if _, found := d.Prefixes[p]; found { - return len(prefix) - i + if d.Prefixes != nil { + for _, p := range d.Prefixes { + if strings.Index(prefix, p) == 0 { + return len(p) + } } } - return + if d.longPrefixesMap != nil { + for i, p := range utils.SplitPrefix(prefix) { + if _, found := d.longPrefixesMap[p]; found { + return len(prefix) - i + } + } + } + return 0 } func (d *Destination) String() (result string) { result = d.Id + ": " - for k, _ := range d.Prefixes { - result += k + ", " + if d.Prefixes != nil { + for _, k := range d.Prefixes { + result += k + ", " + } + } + if d.longPrefixesMap != nil { + for k, _ := range d.longPrefixesMap { + result += k + ", " + } } result = strings.TrimRight(result, ", ") return result } + +func (d *Destination) AddPrefix(pfx string) { + d.Prefixes = append(d.Prefixes, pfx) +} + +func (d *Destination) OptimizePrefixes() { + if len(d.Prefixes) > LONG_PREFIX_SLICE_LENGTH { + d.longPrefixesMap = make(map[string]interface{}) + for _, p := range d.Prefixes { + d.longPrefixesMap[p] = nil + } + d.Prefixes = nil + } +} diff --git a/engine/destinations_test.go b/engine/destinations_test.go index 004d12141..f0218fd35 100644 --- a/engine/destinations_test.go +++ b/engine/destinations_test.go @@ -20,6 +20,7 @@ package engine import ( "encoding/json" + "strconv" "github.com/cgrates/cgrates/cache2go" @@ -27,7 +28,7 @@ import ( ) func TestDestinationStoreRestore(t *testing.T) { - nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}} + nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}} s, _ := json.Marshal(nationale) d1 := &Destination{Id: "nat"} json.Unmarshal(s, d1) @@ -38,22 +39,19 @@ func TestDestinationStoreRestore(t *testing.T) { } func TestDestinationStorageStore(t *testing.T) { - nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}} + nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}} err := storageGetter.SetDestination(nationale) if err != nil { t.Error("Error storing destination: ", err) } result, err := storageGetter.GetDestination(nationale.Id, false) - _, a := nationale.Prefixes["0257"] - _, b := nationale.Prefixes["0256"] - _, c := nationale.Prefixes["0723"] - if !a || !b || !c { + if nationale.containsPrefix("0257") == 0 || nationale.containsPrefix("0256") == 0 || nationale.containsPrefix("0723") == 0 { t.Errorf("Expected %q was %q", nationale, result) } } func TestDestinationContainsPrefix(t *testing.T) { - nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}} + nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}} precision := nationale.containsPrefix("0256") if precision != len("0256") { t.Error("Should contain prefix: ", nationale) @@ -61,7 +59,7 @@ func TestDestinationContainsPrefix(t *testing.T) { } func TestDestinationContainsPrefixLong(t *testing.T) { - nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}} + nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}} precision := nationale.containsPrefix("0256723045") if precision != len("0256") { t.Error("Should contain prefix: ", nationale) @@ -69,7 +67,7 @@ func TestDestinationContainsPrefixLong(t *testing.T) { } func TestDestinationContainsPrefixWrong(t *testing.T) { - nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}} + nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}} precision := nationale.containsPrefix("01234567") if precision != 0 { t.Error("Should not contain prefix: ", nationale) @@ -104,59 +102,32 @@ func TestDestinationGetNotExistsCache(t *testing.T) { } } -/* -func TestConcurrentDestReadWrite(t *testing.T) { - dst1 := &Destination{Id: "TST_1", Prefixes: []string{"1"}} - err := storageGetter.SetDestination(dst1) - if err != nil { - t.Error("Error setting destination: ", err) +func TestDestinationOptimzeShort(t *testing.T) { + d := &Destination{} + for i := 0; i < LONG_PREFIX_SLICE_LENGTH; i++ { + d.AddPrefix(strconv.Itoa(i)) } - rec := 500 - go func() { - for i := 0; i < rec; i++ { - storageGetter.SetDestination(&Destination{Id: fmt.Sprintf("TST_%d", i), Prefixes: []string{"1"}}) - } - }() - - for i := 0; i < rec; i++ { - dst2, err := storageGetter.GetDestination(dst1.Id) - if err != nil { - t.Error("Error retrieving destination: ", err) - } - if !reflect.DeepEqual(dst1, dst2) { - t.Error("Cannot retrieve properly the destination 1", dst1, dst2) - } + d.OptimizePrefixes() + if d.Prefixes == nil || d.longPrefixesMap != nil { + t.Logf("Error optimizing destinations %+v", d) } } -func TestNonConcurrentDestReadWrite(t *testing.T) { - dst1 := &Destination{Id: "TST_1", Prefixes: []string{"1"}} - err := storageGetter.SetDestination(dst1) - if err != nil { - t.Error("Error setting destination: ", err) +func TestDestinationOptimzeLong(t *testing.T) { + d := &Destination{} + for i := 0; i < LONG_PREFIX_SLICE_LENGTH+1; i++ { + d.AddPrefix(strconv.Itoa(i)) } - rec := 10000 - //go func(){ - for i := 0; i < rec; i++ { - storageGetter.SetDestination(&Destination{Id: fmt.Sprintf("TST_%d", i), Prefixes: []string{"1"}}) - } - //}() - - for i := 0; i < rec; i++ { - dst2, err := storageGetter.GetDestination(dst1.Id) - if err != nil { - t.Error("Error retrieving destination: ", err) - } - if !reflect.DeepEqual(dst1, dst2) { - t.Error("Cannot retrieve properly the destination 1", dst1, dst2) - } + d.OptimizePrefixes() + if d.Prefixes != nil || d.longPrefixesMap == nil { + t.Logf("Error optimizing destinations %+v", d) } } -*/ + /********************************* Benchmarks **********************************/ func BenchmarkDestinationStorageStoreRestore(b *testing.B) { - nationale := &Destination{Id: "nat", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}} + nationale := &Destination{Id: "nat", Prefixes: []string{"0257", "0256", "0723"}} for i := 0; i < b.N; i++ { storageGetter.SetDestination(nationale) storageGetter.GetDestination(nationale.Id, true) diff --git a/engine/history_test.go b/engine/history_test.go index 71f98110e..a432493dd 100644 --- a/engine/history_test.go +++ b/engine/history_test.go @@ -31,3 +31,20 @@ func TestHistoryRatinPlans(t *testing.T) { t.Error("Error in destination history content:", scribe.RpBuf.String()) } } + +func TestHistoryDestinations(t *testing.T) { + scribe := historyScribe.(*history.MockScribe) + expected := `[{"Key":"ALL","Object":{"Id":"ALL","Prefixes":["49","41","43"]}} +{"Key":"GERMANY","Object":{"Id":"GERMANY","Prefixes":["49"]}} +{"Key":"GERMANY_O2","Object":{"Id":"GERMANY_O2","Prefixes":["41"]}} +{"Key":"GERMANY_PREMIUM","Object":{"Id":"GERMANY_PREMIUM","Prefixes":["43"]}} +{"Key":"NAT","Object":{"Id":"NAT","Prefixes":["0256","0257","0723"]}} +{"Key":"PSTN_70","Object":{"Id":"PSTN_70","Prefixes":["+4970"]}} +{"Key":"PSTN_71","Object":{"Id":"PSTN_71","Prefixes":["+4971"]}} +{"Key":"PSTN_72","Object":{"Id":"PSTN_72","Prefixes":["+4972"]}} +{"Key":"RET","Object":{"Id":"RET","Prefixes":["0723","0724"]}} +{"Key":"nat","Object":{"Id":"nat","Prefixes":["0257","0256","0723"]}}]` + if scribe.DestBuf.String() != expected { + t.Error("Error in destination history content:", scribe.DestBuf.String()) + } +} diff --git a/engine/loader_csv.go b/engine/loader_csv.go index f1d315e91..7717cbdee 100644 --- a/engine/loader_csv.go +++ b/engine/loader_csv.go @@ -199,10 +199,10 @@ func (csvr *CSVReader) LoadDestinations() (err error) { } } if dest == nil { - dest = &Destination{Id: tag, Prefixes: make(map[string]interface{}, 1)} + dest = &Destination{Id: tag} csvr.destinations = append(csvr.destinations, dest) } - dest.Prefixes[record[1]] = nil + dest.AddPrefix(record[1]) } return } diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 3c910144d..196e1c744 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -151,39 +151,39 @@ func TestLoadDestinations(t *testing.T) { for _, d := range csvr.destinations { switch d.Id { case "NAT": - if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`0256`: nil, `0257`: nil, `0723`: nil}) { + if !reflect.DeepEqual(d.Prefixes, []string{`0256`, `0257`, `0723`}) { t.Error("Faild to load destinations", d) } case "ALL": - if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`49`: nil, `41`: nil, `43`: nil}) { + if !reflect.DeepEqual(d.Prefixes, []string{`49`, `41`, `43`}) { t.Error("Faild to load destinations", d) } case "RET": - if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`0723`: nil, `0724`: nil}) { + if !reflect.DeepEqual(d.Prefixes, []string{`0723`, `0724`}) { t.Error("Faild to load destinations", d) } case "GERMANY": - if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`49`: nil}) { + if !reflect.DeepEqual(d.Prefixes, []string{`49`}) { t.Error("Faild to load destinations", d) } case "GERMANY_O2": - if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`41`: nil}) { + if !reflect.DeepEqual(d.Prefixes, []string{`41`}) { t.Error("Faild to load destinations", d) } case "GERMANY_PREMIUM": - if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`43`: nil}) { + if !reflect.DeepEqual(d.Prefixes, []string{`43`}) { t.Error("Faild to load destinations", d) } case "PSTN_71": - if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`+4971`: nil}) { + if !reflect.DeepEqual(d.Prefixes, []string{`+4971`}) { t.Error("Faild to load destinations", d) } case "PSTN_72": - if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`+4972`: nil}) { + if !reflect.DeepEqual(d.Prefixes, []string{`+4972`}) { t.Error("Faild to load destinations", d) } case "PSTN_70": - if !reflect.DeepEqual(d.Prefixes, map[string]interface{}{`+4970`: nil}) { + if !reflect.DeepEqual(d.Prefixes, []string{`+4970`}) { t.Error("Faild to load destinations", d) } default: diff --git a/engine/storage_map.go b/engine/storage_map.go index a318c29f6..bb8726426 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -158,6 +158,7 @@ func (ms *MapStorage) GetDestination(key string, checkDb bool) (dest *Destinatio if values, ok := ms.dict[key]; ok { dest = &Destination{Id: key} err = ms.ms.Unmarshal(values, dest) + dest.OptimizePrefixes() cache2go.Cache(key, dest) } else { return nil, errors.New("not found") diff --git a/engine/storage_redis.go b/engine/storage_redis.go index a5c792b16..c3dac36b6 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -244,6 +244,7 @@ func (rs *RedisStorage) GetDestination(key string, checkDb bool) (dest *Destinat r.Close() dest = new(Destination) err = rs.ms.Unmarshal(out, dest) + dest.OptimizePrefixes() cache2go.Cache(key, dest) } return diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 3f57958d6..3efc95a1f 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -164,7 +164,7 @@ func (self *SQLStorage) GetTPDestination(tpid, destTag string) (*Destination, er return nil, err } defer rows.Close() - d := &Destination{Id: destTag, Prefixes: make(map[string]interface{}, 1)} + d := &Destination{Id: destTag} i := 0 for rows.Next() { i++ //Keep here a reference so we know we got at least one prefix @@ -173,7 +173,7 @@ func (self *SQLStorage) GetTPDestination(tpid, destTag string) (*Destination, er if err != nil { return nil, err } - d.Prefixes[pref] = nil + d.AddPrefix(pref) } if i == 0 { return nil, nil @@ -804,10 +804,10 @@ func (self *SQLStorage) GetTpDestinations(tpid, tag string) ([]*Destination, err } } if dest == nil { - dest = &Destination{Id: tag, Prefixes: make(map[string]interface{}, 1)} + dest = &Destination{Id: tag} dests = append(dests, dest) } - dest.Prefixes[prefix] = nil + dest.AddPrefix(prefix) } return dests, nil } diff --git a/engine/storage_test.go b/engine/storage_test.go index dfdd3c458..bf91f0fd5 100644 --- a/engine/storage_test.go +++ b/engine/storage_test.go @@ -96,13 +96,13 @@ func TestStorageDestinationContainsPrefixNotExisting(t *testing.T) { } func TestPreCacheRefresh(t *testing.T) { - storageGetter.SetDestination(&Destination{"T11", map[string]interface{}{"0": nil}}) + storageGetter.SetDestination(&Destination{"T11", []string{"0"}, nil}) storageGetter.GetDestination("T11", false) - storageGetter.SetDestination(&Destination{"T11", map[string]interface{}{"1": nil}}) + storageGetter.SetDestination(&Destination{"T11", []string{"1"}, nil}) storageGetter.PreCache(nil, nil, nil, nil) d, err := storageGetter.GetDestination("T11", false) - _, ok := d.Prefixes["1"] - if err != nil || !ok { + p := d.containsPrefix("1") + if err != nil || p == 0 { t.Error("Error refreshing cache:", d) } } diff --git a/engine/tpimporter_csv.go b/engine/tpimporter_csv.go index e0d7b3165..db63d1bcf 100644 --- a/engine/tpimporter_csv.go +++ b/engine/tpimporter_csv.go @@ -115,7 +115,7 @@ func (self *TPCSVImporter) importDestinations(fn string) error { } continue } - dst := &Destination{record[0], map[string]interface{}{record[1]: nil}} + dst := &Destination{record[0], []string{record[1]}, nil} if err := self.StorDb.SetTPDestination(self.TPid, dst); err != nil { if self.Verbose { log.Printf("Ignoring line %d, storDb operational error: <%s> ", lineNr, err.Error()) diff --git a/engine/userbalance_test.go b/engine/userbalance_test.go index 715d43e9b..766820dbe 100644 --- a/engine/userbalance_test.go +++ b/engine/userbalance_test.go @@ -24,8 +24,8 @@ import ( ) var ( - NAT = &Destination{Id: "NAT", Prefixes: map[string]interface{}{"0257": nil, "0256": nil, "0723": nil}} - RET = &Destination{Id: "RET", Prefixes: map[string]interface{}{"0723": nil, "0724": nil}} + NAT = &Destination{Id: "NAT", Prefixes: []string{"0257", "0256", "0723"}} + RET = &Destination{Id: "RET", Prefixes: []string{"0723", "0724"}} ) func init() { diff --git a/utils/apitpdata.go b/utils/apitpdata.go index eb0f3a7b9..ca340ffb7 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -26,9 +26,9 @@ import ( ) type TPDestination struct { - TPid string // Tariff plan id - DestinationId string // Destination id - Prefixes map[string]interface{} // Prefixes attached to this destination + TPid string // Tariff plan id + DestinationId string // Destination id + Prefixes []string // Prefixes attached to this destination } // This file deals with tp_* data definition