From 450d2e50a35ed2cdea8783b2e80839d61de8eb7b Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 17 Feb 2012 16:36:10 +0200 Subject: [PATCH] migrating storage adapters to new structure --- cmd/loader/loader.go | 26 +++--- timespans/activationperiod.go | 56 ------------- timespans/calldesc.go | 24 +----- timespans/calldesc_test.go | 30 +++---- timespans/kyoto_storage.go | 84 +++++++++++++++++-- ...onperiod_test.go => kyoto_storage_test.go} | 12 +-- timespans/mongo_storage.go | 27 +++--- timespans/redis_storage.go | 83 ++++++++++++++++-- timespans/storage_interface.go | 3 +- 9 files changed, 206 insertions(+), 139 deletions(-) rename timespans/{activationperiod_test.go => kyoto_storage_test.go} (86%) diff --git a/cmd/loader/loader.go b/cmd/loader/loader.go index a5c3b42e1..c73358666 100644 --- a/cmd/loader/loader.go +++ b/cmd/loader/loader.go @@ -2,9 +2,6 @@ package main import ( "flag" - "github.com/simonz05/godis" - "github.com/fsouza/gokabinet/kc" - "launchpad.net/mgo" "github.com/rif/cgrates/timespans" "log" "os" @@ -45,14 +42,13 @@ func main() { switch *storage { case "kyoto": - db, _ := kc.Open(*kyotofile, kc.WRITE) - defer db.Close() + storage, _ := timespans.NewKyotoStorage(*kyotofile) + defer storage.Close() for _, cd := range callDescriptors{ - key := cd.GetKey() - db.Set(key, cd.EncodeValues()) - log.Printf("Storing %q", key) + storage.SetActivationPeriods(cd.GetKey(), cd.ActivationPeriods) + log.Printf("Storing %q", cd.GetKey()) } - case "mongo": + /*case "mongo": session, err := mgo.Dial(*mongoserver) if err != nil { panic(err) @@ -63,17 +59,17 @@ func main() { c := session.DB(*mongodb).C("ap") for _, cd := range callDescriptors{ key := cd.GetKey() - c.Insert(×pans.KeyValue{key, cd.EncodeValues()}) + c.Insert(&map[string]string{"_id":key, "value":cd.EncodeValues()}) log.Printf("Storing %q", key) - } + }*/ default: - db := godis.New(*redisserver, *redisdb, *redispass) - defer db.Quit() + storage, _ := timespans.NewKyotoStorage(*redisserver, *redisdb) + defer storage.Close() for _, cd := range callDescriptors{ key := cd.GetKey() - db.Set(key, cd.EncodeValues()) - log.Printf("Storing %q", key) + storage.SetActivationPeriods(cd.GetKey(), cd.ActivationPeriods) + log.Printf("Storing %q", cd.GetKey()) } } } diff --git a/timespans/activationperiod.go b/timespans/activationperiod.go index 3745c74ef..1d51b1801 100644 --- a/timespans/activationperiod.go +++ b/timespans/activationperiod.go @@ -1,8 +1,6 @@ package timespans import ( - "strconv" - "strings" "time" //"log" ) @@ -23,57 +21,3 @@ func (ap *ActivationPeriod) AddInterval(is ...*Interval) { ap.Intervals = append(ap.Intervals, i) } } - -/* -Serializes the objects for the storage. -*/ -func (ap *ActivationPeriod) store() (result string) { - result += strconv.FormatInt(ap.ActivationTime.UnixNano(), 10) + ";" - var is string - for _, i := range ap.Intervals { - is = strconv.Itoa(int(i.Month)) + "|" - is += strconv.Itoa(i.MonthDay) + "|" - for _, wd := range i.WeekDays { - is += strconv.Itoa(int(wd)) + "," - } - is = strings.TrimRight(is, ",") + "|" - is += i.StartTime + "|" - is += i.EndTime + "|" - is += strconv.FormatFloat(i.Ponder, 'f', -1, 64) + "|" - is += strconv.FormatFloat(i.ConnectFee, 'f', -1, 64) + "|" - is += strconv.FormatFloat(i.Price, 'f', -1, 64) + "|" - is += strconv.FormatFloat(i.BillingUnit, 'f', -1, 64) - result += is + ";" - } - return -} - -/* -De-serializes the objects for the storage. -*/ -func (ap *ActivationPeriod) restore(input string) { - elements := strings.Split(input, ";") - unixNano, _ := strconv.ParseInt(elements[0], 10, 64) - ap.ActivationTime = time.Unix(0, unixNano).In(time.UTC) - for _, is := range elements[1 : len(elements)-1] { - i := &Interval{} - ise := strings.Split(is, "|") - month, _ := strconv.Atoi(ise[0]) - i.Month = time.Month(month) - i.MonthDay, _ = strconv.Atoi(ise[1]) - for _, d := range strings.Split(ise[2], ",") { - if d != "" { - wd, _ := strconv.Atoi(d) - i.WeekDays = append(i.WeekDays, time.Weekday(wd)) - } - } - i.StartTime = ise[3] - i.EndTime = ise[4] - i.Ponder, _ = strconv.ParseFloat(ise[5], 64) - i.ConnectFee, _ = strconv.ParseFloat(ise[6], 64) - i.Price, _ = strconv.ParseFloat(ise[7], 64) - i.BillingUnit, _ = strconv.ParseFloat(ise[8], 64) - - ap.Intervals = append(ap.Intervals, i) - } -} diff --git a/timespans/calldesc.go b/timespans/calldesc.go index 057a24d46..5292facde 100644 --- a/timespans/calldesc.go +++ b/timespans/calldesc.go @@ -4,7 +4,6 @@ import ( "fmt" "log" "math" - "strings" "time" ) @@ -44,17 +43,6 @@ func (cd *CallDescriptor) AddActivationPeriod(aps ...*ActivationPeriod) { } } -/* -Creates a string ready for storage containing the serialization of all -activation periods held in the internal list. -*/ -func (cd *CallDescriptor) EncodeValues() (result string) { - for _, ap := range cd.ActivationPeriods { - result += ap.store() + "\n" - } - return -} - /* Restores the activation periods from storage. */ @@ -63,22 +51,16 @@ func (cd *CallDescriptor) RestoreFromStorage(sg StorageGetter) (destPrefix strin base := fmt.Sprintf("%s:%s:", cd.CstmId, cd.Subject) destPrefix = cd.DestinationPrefix key := base + destPrefix - values, err := sg.Get(key) + values, err := sg.GetActivationPeriods(key) //get for a smaller prefix if the orignal one was not found - for i := len(cd.DestinationPrefix); err != nil && i > 1; values, err = sg.Get(key) { + for i := len(cd.DestinationPrefix); err != nil && i > 1; values, err = sg.GetActivationPeriods(key) { i-- destPrefix = cd.DestinationPrefix[:i] key = base + destPrefix } //load the activation preriods if err == nil { - for _, aps := range strings.Split(values, "\n") { - if len(aps) > 0 { - ap := &ActivationPeriod{} - ap.restore(aps) - cd.ActivationPeriods = append(cd.ActivationPeriods, ap) - } - } + cd.ActivationPeriods = values } return } diff --git a/timespans/calldesc_test.go b/timespans/calldesc_test.go index c569b3b48..7d717ef03 100644 --- a/timespans/calldesc_test.go +++ b/timespans/calldesc_test.go @@ -162,7 +162,7 @@ func BenchmarkRedisGetting(b *testing.B) { cd := &CallDescriptor{CstmId: "vdf", Subject: "rif", DestinationPrefix: "0256", TimeStart: t1, TimeEnd: t2} b.StartTimer() for i := 0; i < b.N; i++ { - getter.Get(cd.GetKey()) + getter.GetActivationPeriods(cd.GetKey()) } } @@ -205,7 +205,7 @@ func BenchmarkKyotoGetting(b *testing.B) { b.StartTimer() for i := 0; i < b.N; i++ { key := cd.GetKey() - getter.Get(key) + getter.GetActivationPeriods(key) } } @@ -254,7 +254,7 @@ func BenchmarkKyotoGetCost(b *testing.B) { func BenchmarkMongoGetting(b *testing.B) { b.StopTimer() - getter, _ := NewMongoStorage("127.0.0.1","test") + getter, _ := NewMongoStorage("127.0.0.1", "test") defer getter.Close() t1 := time.Date(2012, time.February, 2, 17, 30, 0, 0, time.UTC) @@ -267,16 +267,16 @@ func BenchmarkMongoGetting(b *testing.B) { } } -func BenchmarkMongoGetCost(b *testing.B) { - b.StopTimer() - getter, _ := NewMongoStorage("127.0.0.1","test") - defer getter.Close() +//func BenchmarkMongoGetCost(b *testing.B) { +// b.StopTimer() +// getter, _ := NewMongoStorage("127.0.0.1", "test") +// defer getter.Close() - t1 := time.Date(2012, time.February, 2, 17, 30, 0, 0, time.UTC) - t2 := time.Date(2012, time.February, 2, 18, 30, 0, 0, time.UTC) - cd := &CallDescriptor{CstmId: "vdf", Subject: "rif", DestinationPrefix: "0256", TimeStart: t1, TimeEnd: t2} - b.StartTimer() - for i := 0; i < b.N; i++ { - cd.GetCost(getter) - } -} +// t1 := time.Date(2012, time.February, 2, 17, 30, 0, 0, time.UTC) +// t2 := time.Date(2012, time.February, 2, 18, 30, 0, 0, time.UTC) +// cd := &CallDescriptor{CstmId: "vdf", Subject: "rif", DestinationPrefix: "0256", TimeStart: t1, TimeEnd: t2} +// b.StartTimer() +// for i := 0; i < b.N; i++ { +// cd.GetCost(getter) +// } +//} diff --git a/timespans/kyoto_storage.go b/timespans/kyoto_storage.go index ff5517292..18fcf2133 100644 --- a/timespans/kyoto_storage.go +++ b/timespans/kyoto_storage.go @@ -1,8 +1,10 @@ package timespans import ( - //"log" "github.com/fsouza/gokabinet/kc" + "time" + "strconv" + "strings" ) type KyotoStorage struct { @@ -11,15 +13,87 @@ type KyotoStorage struct { func NewKyotoStorage(filaName string) (*KyotoStorage, error) { ndb, err := kc.Open(filaName, kc.READ) - //log.Print("Starting kyoto storage") return &KyotoStorage{db: ndb}, err } func (ks *KyotoStorage) Close() { - //log.Print("Closing kyoto storage") ks.db.Close() } -func (ks *KyotoStorage) Get(key string) (value string, err error) { - return ks.db.Get(key) +func (ks *KyotoStorage) GetActivationPeriods(key string) (aps []*ActivationPeriod, err error) { + values, err := ks.db.Get(key) + + if err == nil { + for _, ap_string := range strings.Split(values, "\n") { + if len(ap_string) > 0 { + ap := ks.restore(ap_string) + aps = append(aps, ap) + } + } + } + return aps, err +} + +func (ks *KyotoStorage) SetActivationPeriods(key string, aps []*ActivationPeriod){ + result := "" + for _, ap := range aps { + result += ks.store(ap) + "\n" + } + ks.db.Set(key, result) +} + +/* +Serializes the activation periods for the storage. +*/ +func (ks *KyotoStorage) store(ap *ActivationPeriod) (result string) { + result += strconv.FormatInt(ap.ActivationTime.UnixNano(), 10) + ";" + var is string + for _, i := range ap.Intervals { + is = strconv.Itoa(int(i.Month)) + "|" + is += strconv.Itoa(i.MonthDay) + "|" + for _, wd := range i.WeekDays { + is += strconv.Itoa(int(wd)) + "," + } + is = strings.TrimRight(is, ",") + "|" + is += i.StartTime + "|" + is += i.EndTime + "|" + is += strconv.FormatFloat(i.Ponder, 'f', -1, 64) + "|" + is += strconv.FormatFloat(i.ConnectFee, 'f', -1, 64) + "|" + is += strconv.FormatFloat(i.Price, 'f', -1, 64) + "|" + is += strconv.FormatFloat(i.BillingUnit, 'f', -1, 64) + result += is + ";" + } + return +} + +/* +De-serializes the activation periods for the storage. +*/ +func (ks *KyotoStorage) restore(input string) (ap *ActivationPeriod) { + elements := strings.Split(input, ";") + unixNano, _ := strconv.ParseInt(elements[0], 10, 64) + ap = &ActivationPeriod{} + ap.ActivationTime = time.Unix(0, unixNano).In(time.UTC) + for _, is := range elements[1 : len(elements)-1] { + i := &Interval{} + ise := strings.Split(is, "|") + month, _ := strconv.Atoi(ise[0]) + i.Month = time.Month(month) + i.MonthDay, _ = strconv.Atoi(ise[1]) + for _, d := range strings.Split(ise[2], ",") { + if d != "" { + wd, _ := strconv.Atoi(d) + i.WeekDays = append(i.WeekDays, time.Weekday(wd)) + } + } + i.StartTime = ise[3] + i.EndTime = ise[4] + i.Ponder, _ = strconv.ParseFloat(ise[5], 64) + i.ConnectFee, _ = strconv.ParseFloat(ise[6], 64) + i.Price, _ = strconv.ParseFloat(ise[7], 64) + i.BillingUnit, _ = strconv.ParseFloat(ise[8], 64) + + ap.Intervals = append(ap.Intervals, i) + } + return } diff --git a/timespans/activationperiod_test.go b/timespans/kyoto_storage_test.go similarity index 86% rename from timespans/activationperiod_test.go rename to timespans/kyoto_storage_test.go index 7fc152a82..e279ce530 100644 --- a/timespans/activationperiod_test.go +++ b/timespans/kyoto_storage_test.go @@ -13,15 +13,15 @@ func TestApStoreRestore(t *testing.T) { WeekDays: []time.Weekday{time.Wednesday, time.Thursday}, StartTime: "14:30:00", EndTime: "15:00:00"} - ap := ActivationPeriod{ActivationTime: d} + ap := &ActivationPeriod{ActivationTime: d} ap.AddInterval(i) - result := ap.store() + storage, _ := NewKyotoStorage("test.kch") + result := storage.store(ap) expected := "1328106601000000000;2|1|3,4|14:30:00|15:00:00|0|0|0|0;" if result != expected { t.Errorf("Expected %q was %q", expected, result) } - ap1 := ActivationPeriod{} - ap1.restore(result) + ap1 := storage.restore(result) if ap1.ActivationTime != ap.ActivationTime { t.Errorf("Expected %v was %v", ap.ActivationTime, ap1.ActivationTime) } @@ -58,8 +58,8 @@ func TestApStoreRestore(t *testing.T) { } func BenchmarkActivationPeriodRestore(b *testing.B) { - ap1 := ActivationPeriod{} + storage, _ := NewKyotoStorage("test.kch") for i := 0; i < b.N; i++ { - ap1.restore("1328106601;2|1|3,4|14:30:00|15:00:00|0|0|0|0;") + storage.restore("1328106601;2|1|3,4|14:30:00|15:00:00|0|0|0|0;") } } diff --git a/timespans/mongo_storage.go b/timespans/mongo_storage.go index de42c0354..e0288e67c 100644 --- a/timespans/mongo_storage.go +++ b/timespans/mongo_storage.go @@ -1,40 +1,37 @@ package timespans import ( - "launchpad.net/mgo/bson" - "launchpad.net/mgo" + "launchpad.net/mgo" + "launchpad.net/mgo/bson" ) type KeyValue struct { - Key string - Value string + _id string + value string } type MongoStorage struct { - db *mgo.Collection + db *mgo.Collection session *mgo.Session } func NewMongoStorage(address, db string) (*MongoStorage, error) { session, err := mgo.Dial(address) - if err != nil { - panic(err) - } - session.SetMode(mgo.Monotonic, true) + if err != nil { + panic(err) + } + session.SetMode(mgo.Monotonic, true) - ndb := session.DB(db).C("ap") - //log.Print("Starting redis storage") + ndb := session.DB(db).C("ap") return &MongoStorage{db: ndb, session: session}, nil } func (ms *MongoStorage) Close() { - //log.Print("Closing redis storage") ms.session.Close() } func (ms *MongoStorage) Get(key string) (string, error) { result := KeyValue{} - err := ms.db.Find(bson.M{"key": key}).One(&result) - - return result.Value, err + err := ms.db.Find(bson.M{"_id": key}).One(&result) + return result.value, err } diff --git a/timespans/redis_storage.go b/timespans/redis_storage.go index a43d5c7b8..1ede4befa 100644 --- a/timespans/redis_storage.go +++ b/timespans/redis_storage.go @@ -1,8 +1,10 @@ package timespans import ( + "strings" + "strconv" + "time" "github.com/simonz05/godis" - //"log" ) type RedisStorage struct { @@ -11,16 +13,87 @@ type RedisStorage struct { func NewRedisStorage(address string, db int) (*RedisStorage, error) { ndb := godis.New(address, db, "") - //log.Print("Starting redis storage") return &RedisStorage{db: ndb}, nil } func (rs *RedisStorage) Close() { - //log.Print("Closing redis storage") rs.db.Quit() } -func (rs *RedisStorage) Get(key string) (string, error) { +func (rs *RedisStorage) GetActivationPeriods(key string) (aps []*ActivationPeriod, err error) { elem, err := rs.db.Get(key) - return elem.String(), err + values:= elem.String() + if err == nil { + for _, ap_string := range strings.Split(values, "\n") { + if len(ap_string) > 0 { + ap := rs.restore(ap_string) + aps = append(aps, ap) + } + } + } + return aps, err +} + +func (rs *RedisStorage) SetActivationPeriods(key string, aps []*ActivationPeriod){ + result := "" + for _, ap := range aps { + result += rs.store(ap) + "\n" + } + rs.db.Set(key, result) +} + +/* +Serializes the activation periods for the storage. +*/ +func (rs *RedisStorage) store(ap *ActivationPeriod) (result string) { + result += strconv.FormatInt(ap.ActivationTime.UnixNano(), 10) + ";" + var is string + for _, i := range ap.Intervals { + is = strconv.Itoa(int(i.Month)) + "|" + is += strconv.Itoa(i.MonthDay) + "|" + for _, wd := range i.WeekDays { + is += strconv.Itoa(int(wd)) + "," + } + is = strings.TrimRight(is, ",") + "|" + is += i.StartTime + "|" + is += i.EndTime + "|" + is += strconv.FormatFloat(i.Ponder, 'f', -1, 64) + "|" + is += strconv.FormatFloat(i.ConnectFee, 'f', -1, 64) + "|" + is += strconv.FormatFloat(i.Price, 'f', -1, 64) + "|" + is += strconv.FormatFloat(i.BillingUnit, 'f', -1, 64) + result += is + ";" + } + return +} + +/* +De-serializes the activation periods for the storage. +*/ +func (rs *RedisStorage) restore(input string) (ap *ActivationPeriod) { + elements := strings.Split(input, ";") + unixNano, _ := strconv.ParseInt(elements[0], 10, 64) + ap = &ActivationPeriod{} + ap.ActivationTime = time.Unix(0, unixNano).In(time.UTC) + for _, is := range elements[1 : len(elements)-1] { + i := &Interval{} + ise := strings.Split(is, "|") + month, _ := strconv.Atoi(ise[0]) + i.Month = time.Month(month) + i.MonthDay, _ = strconv.Atoi(ise[1]) + for _, d := range strings.Split(ise[2], ",") { + if d != "" { + wd, _ := strconv.Atoi(d) + i.WeekDays = append(i.WeekDays, time.Weekday(wd)) + } + } + i.StartTime = ise[3] + i.EndTime = ise[4] + i.Ponder, _ = strconv.ParseFloat(ise[5], 64) + i.ConnectFee, _ = strconv.ParseFloat(ise[6], 64) + i.Price, _ = strconv.ParseFloat(ise[7], 64) + i.BillingUnit, _ = strconv.ParseFloat(ise[8], 64) + + ap.Intervals = append(ap.Intervals, i) + } + return } diff --git a/timespans/storage_interface.go b/timespans/storage_interface.go index 08fcdd4a8..3f8c8d659 100644 --- a/timespans/storage_interface.go +++ b/timespans/storage_interface.go @@ -5,5 +5,6 @@ Interface for storage providers. */ type StorageGetter interface { Close() - Get(key string) (string, error) + GetActivationPeriods(key string) ([]*ActivationPeriod, error) + SetActivationPeriods(key string, aps []*ActivationPeriod) }