diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 327c33e66..34f2bd36e 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -720,7 +720,7 @@ func main() { } defer dataDB.Close() engine.SetDataStorage(dataDB) - if err := engine.CheckVersions(nil); err != nil { + if err := engine.CheckVersions(dataDB); err != nil { fmt.Println(err.Error()) return } @@ -737,6 +737,10 @@ func main() { loadDb = storDb.(engine.LoadStorage) cdrDb = storDb.(engine.CdrStorage) engine.SetCdrStorage(cdrDb) + if err := engine.CheckVersions(storDb); err != nil { + fmt.Println(err.Error()) + return + } } engine.SetRoundingDecimals(cfg.RoundingDecimals) diff --git a/cmd/cgr-loader/migrator_mongo.go b/cmd/cgr-loader/migrator_mongo.go deleted file mode 100644 index 85400e7a8..000000000 --- a/cmd/cgr-loader/migrator_mongo.go +++ /dev/null @@ -1,114 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package main - -import ( - "fmt" - "log" - - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -type MongoMigrator struct { - session *mgo.Session - db *mgo.Database -} - -func NewMongoMigrator(host, port, db, user, pass string) (*MongoMigrator, error) { - address := fmt.Sprintf("%s:%s", host, port) - if user != "" && pass != "" { - address = fmt.Sprintf("%s:%s@%s", user, pass, address) - } - session, err := mgo.Dial(address) - if err != nil { - return nil, err - } - ndb := session.DB(db) - return &MongoMigrator{session: session, db: ndb}, nil -} - -func (mig MongoMigrator) migrateActions() error { - newAcsMap := make(map[string]engine.Actions) - iter := mig.db.C("actions").Find(nil).Iter() - var oldAcs struct { - Key string - Value Actions2 - } - for iter.Next(&oldAcs) { - log.Printf("Migrating action: %s...", oldAcs.Key) - newAcs := make(engine.Actions, len(oldAcs.Value)) - for index, oldAc := range oldAcs.Value { - a := &engine.Action{ - Id: oldAc.Id, - ActionType: oldAc.ActionType, - ExtraParameters: oldAc.ExtraParameters, - ExpirationString: oldAc.ExpirationString, - Filter: oldAc.Filter, - Weight: oldAc.Weight, - Balance: &engine.BalanceFilter{ - Uuid: oldAc.Balance.Uuid, - ID: oldAc.Balance.ID, - Type: oldAc.Balance.Type, - Directions: oldAc.Balance.Directions, - ExpirationDate: oldAc.Balance.ExpirationDate, - Weight: oldAc.Balance.Weight, - DestinationIDs: oldAc.Balance.DestinationIDs, - RatingSubject: oldAc.Balance.RatingSubject, - Categories: oldAc.Balance.Categories, - SharedGroups: oldAc.Balance.SharedGroups, - TimingIDs: oldAc.Balance.TimingIDs, - Timings: oldAc.Balance.Timings, - Disabled: oldAc.Balance.Disabled, - Factor: oldAc.Balance.Factor, - Blocker: oldAc.Balance.Blocker, - }, - } - if oldAc.Balance.Value != nil { - a.Balance.Value = &utils.ValueFormula{Static: *oldAc.Balance.Value} - } - newAcs[index] = a - } - newAcsMap[oldAcs.Key] = newAcs - } - if err := iter.Close(); err != nil { - return err - } - - // write data back - for key, acs := range newAcsMap { - if _, err := mig.db.C("actions").Upsert(bson.M{"key": key}, &struct { - Key string - Value engine.Actions - }{Key: key, Value: acs}); err != nil { - return err - } - } - return nil -} - -func (mig MongoMigrator) writeVersion() error { - _, err := mig.db.C("versions").Upsert(bson.M{"key": utils.VERSION_PREFIX + "struct"}, &struct { - Key string - Value *engine.StructVersion - }{utils.VERSION_PREFIX + "struct", engine.CurrentVersion}) - return err -} diff --git a/cmd/cgr-loader/migrator_rc8.go b/cmd/cgr-loader/migrator_rc8.go deleted file mode 100644 index bf3fa3954..000000000 --- a/cmd/cgr-loader/migrator_rc8.go +++ /dev/null @@ -1,696 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package main - -import ( - "bytes" - "compress/zlib" - "fmt" - "log" - "strings" - "time" - - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" - "github.com/mediocregopher/radix.v2/redis" -) - -const OLD_ACCOUNT_PREFIX = "ubl_" - -type MigratorRC8 struct { - db *redis.Client - ms engine.Marshaler -} - -func NewMigratorRC8(address string, db int, pass, mrshlerStr string) (*MigratorRC8, error) { - client, err := redis.Dial("tcp", address) - if err != nil { - return nil, err - } - if err := client.Cmd("SELECT", db).Err; err != nil { - return nil, err - } - if pass != "" { - if err := client.Cmd("AUTH", pass).Err; err != nil { - return nil, err - } - } - - var mrshler engine.Marshaler - if mrshlerStr == utils.MSGPACK { - mrshler = engine.NewCodecMsgpackMarshaler() - } else if mrshlerStr == utils.JSON { - mrshler = new(engine.JSONMarshaler) - } else { - return nil, fmt.Errorf("Unsupported marshaler: %v", mrshlerStr) - } - return &MigratorRC8{db: client, ms: mrshler}, nil -} - -type Account struct { - Id string - BalanceMap map[string]BalanceChain - UnitCounters []*UnitsCounter - ActionTriggers ActionTriggers - AllowNegative bool - Disabled bool -} -type BalanceChain []*Balance - -type Balance struct { - Uuid string //system wide unique - Id string // account wide unique - Value float64 - ExpirationDate time.Time - Weight float64 - DestinationIds string - RatingSubject string - Category string - SharedGroup string - Timings []*engine.RITiming - TimingIDs string - Disabled bool - precision int - account *Account - dirty bool -} - -func (b *Balance) IsDefault() bool { - return (b.DestinationIds == "" || b.DestinationIds == utils.ANY) && - b.RatingSubject == "" && - b.Category == "" && - b.ExpirationDate.IsZero() && - b.SharedGroup == "" && - b.Weight == 0 && - b.Disabled == false -} - -type UnitsCounter struct { - Direction string - BalanceType string - // Units float64 - Balances BalanceChain // first balance is the general one (no destination) -} - -type ActionTriggers []*ActionTrigger - -type ActionTrigger struct { - Id string - ThresholdType string - ThresholdValue float64 - Recurrent bool - MinSleep time.Duration - BalanceId string - BalanceType string - BalanceDirection string - BalanceDestinationIds string - BalanceWeight float64 - BalanceExpirationDate time.Time - BalanceTimingTags string - BalanceRatingSubject string - BalanceCategory string - BalanceSharedGroup string - BalanceDisabled bool - Weight float64 - ActionsId string - MinQueuedItems int - Executed bool -} -type Actions []*Action - -type Action struct { - Id string - ActionType string - BalanceType string - Direction string - ExtraParameters string - ExpirationString string - Weight float64 - Balance *Balance -} - -type ActionPlan struct { - Uuid string // uniquely identify the timing - Id string // informative purpose only - AccountIds []string - Timing *engine.RateInterval - Weight float64 - ActionsId string - actions Actions - stCache time.Time // cached time of the next start -} - -func (at *ActionPlan) IsASAP() bool { - if at.Timing == nil { - return false - } - return at.Timing.Timing.StartTime == utils.ASAP -} - -type SharedGroup struct { - Id string - AccountParameters map[string]*engine.SharingParameters - MemberIds []string -} - -type ActionPlans []*ActionPlan - -func (mig MigratorRC8) migrateAccounts() error { - keys, err := mig.db.Cmd("KEYS", OLD_ACCOUNT_PREFIX+"*").List() - if err != nil { - return err - } - newAccounts := make([]*engine.Account, 0) - var migratedKeys []string - // get existing accounts - for _, key := range keys { - log.Printf("Migrating account: %s...", key) - values, err := mig.db.Cmd("GET", key).Bytes() - if err != nil { - continue - } - var oldAcc Account - if err = mig.ms.Unmarshal(values, &oldAcc); err != nil { - return err - } - // transfer data into new structurse - newAcc := &engine.Account{ - ID: oldAcc.Id, - BalanceMap: make(map[string]engine.Balances, len(oldAcc.BalanceMap)), - UnitCounters: make(engine.UnitCounters, len(oldAcc.UnitCounters)), - ActionTriggers: make(engine.ActionTriggers, len(oldAcc.ActionTriggers)), - AllowNegative: oldAcc.AllowNegative, - Disabled: oldAcc.Disabled, - } - // fix id - idElements := strings.Split(newAcc.ID, utils.CONCATENATED_KEY_SEP) - if len(idElements) != 3 { - log.Printf("Malformed account ID %s", oldAcc.Id) - continue - } - newAcc.ID = fmt.Sprintf("%s:%s", idElements[1], idElements[2]) - // balances - balanceErr := false - for oldBalKey, oldBalChain := range oldAcc.BalanceMap { - keyElements := strings.Split(oldBalKey, "*") - if len(keyElements) != 3 { - log.Printf("Malformed balance key in %s: %s", oldAcc.Id, oldBalKey) - balanceErr = true - break - } - newBalKey := "*" + keyElements[1] - newBalDirection := "*" + keyElements[2] - newAcc.BalanceMap[newBalKey] = make(engine.Balances, len(oldBalChain)) - for index, oldBal := range oldBalChain { - // check default to set new id - if oldBal.IsDefault() { - oldBal.Id = utils.META_DEFAULT - } - newAcc.BalanceMap[newBalKey][index] = &engine.Balance{ - Uuid: oldBal.Uuid, - ID: oldBal.Id, - Value: oldBal.Value, - Directions: utils.ParseStringMap(newBalDirection), - ExpirationDate: oldBal.ExpirationDate, - Weight: oldBal.Weight, - DestinationIDs: utils.ParseStringMap(oldBal.DestinationIds), - RatingSubject: oldBal.RatingSubject, - Categories: utils.ParseStringMap(oldBal.Category), - SharedGroups: utils.ParseStringMap(oldBal.SharedGroup), - Timings: oldBal.Timings, - TimingIDs: utils.ParseStringMap(oldBal.TimingIDs), - Disabled: oldBal.Disabled, - } - } - } - if balanceErr { - continue - } - // unit counters - for _, oldUc := range oldAcc.UnitCounters { - newUc := &engine.UnitCounter{ - Counters: make(engine.CounterFilters, len(oldUc.Balances)), - } - for index, oldUcBal := range oldUc.Balances { - bf := &engine.BalanceFilter{} - if oldUcBal.Uuid != "" { - bf.Uuid = utils.StringPointer(oldUcBal.Uuid) - } - if oldUcBal.Id != "" { - bf.ID = utils.StringPointer(oldUcBal.Id) - } - if oldUc.BalanceType != "" { - bf.Type = utils.StringPointer(oldUc.BalanceType) - } - // the value was used for counter value - /*if oldUcBal.Value != 0 { - bf.Value = utils.Float64Pointer(oldUcBal.Value) - }*/ - if oldUc.Direction != "" { - bf.Directions = utils.StringMapPointer(utils.ParseStringMap(oldUc.Direction)) - } - if !oldUcBal.ExpirationDate.IsZero() { - bf.ExpirationDate = utils.TimePointer(oldUcBal.ExpirationDate) - } - if oldUcBal.Weight != 0 { - bf.Weight = utils.Float64Pointer(oldUcBal.Weight) - } - if oldUcBal.DestinationIds != "" { - bf.DestinationIDs = utils.StringMapPointer(utils.ParseStringMap(oldUcBal.DestinationIds)) - } - if oldUcBal.RatingSubject != "" { - bf.RatingSubject = utils.StringPointer(oldUcBal.RatingSubject) - } - if oldUcBal.Category != "" { - bf.Categories = utils.StringMapPointer(utils.ParseStringMap(oldUcBal.Category)) - } - if oldUcBal.SharedGroup != "" { - bf.SharedGroups = utils.StringMapPointer(utils.ParseStringMap(oldUcBal.SharedGroup)) - } - if oldUcBal.TimingIDs != "" { - bf.TimingIDs = utils.StringMapPointer(utils.ParseStringMap(oldUcBal.TimingIDs)) - } - if oldUcBal.Disabled != false { - bf.Disabled = utils.BoolPointer(oldUcBal.Disabled) - } - bf.Timings = oldUcBal.Timings - cf := &engine.CounterFilter{ - Value: oldUcBal.Value, - Filter: bf, - } - newUc.Counters[index] = cf - } - newAcc.UnitCounters[oldUc.BalanceType] = append(newAcc.UnitCounters[oldUc.BalanceType], newUc) - } - // action triggers - for index, oldAtr := range oldAcc.ActionTriggers { - at := &engine.ActionTrigger{ - UniqueID: oldAtr.Id, - ThresholdType: oldAtr.ThresholdType, - ThresholdValue: oldAtr.ThresholdValue, - Recurrent: oldAtr.Recurrent, - MinSleep: oldAtr.MinSleep, - Weight: oldAtr.Weight, - ActionsID: oldAtr.ActionsId, - MinQueuedItems: oldAtr.MinQueuedItems, - Executed: oldAtr.Executed, - } - bf := &engine.BalanceFilter{} - if oldAtr.BalanceId != "" { - bf.ID = utils.StringPointer(oldAtr.BalanceId) - } - if oldAtr.BalanceType != "" { - bf.Type = utils.StringPointer(oldAtr.BalanceType) - } - if oldAtr.BalanceRatingSubject != "" { - bf.RatingSubject = utils.StringPointer(oldAtr.BalanceRatingSubject) - } - if oldAtr.BalanceDirection != "" { - bf.Directions = utils.StringMapPointer(utils.ParseStringMap(oldAtr.BalanceDirection)) - } - if oldAtr.BalanceDestinationIds != "" { - bf.DestinationIDs = utils.StringMapPointer(utils.ParseStringMap(oldAtr.BalanceDestinationIds)) - } - if oldAtr.BalanceTimingTags != "" { - bf.TimingIDs = utils.StringMapPointer(utils.ParseStringMap(oldAtr.BalanceTimingTags)) - } - if oldAtr.BalanceCategory != "" { - bf.Categories = utils.StringMapPointer(utils.ParseStringMap(oldAtr.BalanceCategory)) - } - if oldAtr.BalanceSharedGroup != "" { - bf.SharedGroups = utils.StringMapPointer(utils.ParseStringMap(oldAtr.BalanceSharedGroup)) - } - if oldAtr.BalanceWeight != 0 { - bf.Weight = utils.Float64Pointer(oldAtr.BalanceWeight) - } - if oldAtr.BalanceDisabled != false { - bf.Disabled = utils.BoolPointer(oldAtr.BalanceDisabled) - } - if !oldAtr.BalanceExpirationDate.IsZero() { - bf.ExpirationDate = utils.TimePointer(oldAtr.BalanceExpirationDate) - } - at.Balance = bf - newAcc.ActionTriggers[index] = at - if newAcc.ActionTriggers[index].ThresholdType == "*min_counter" || - newAcc.ActionTriggers[index].ThresholdType == "*max_counter" { - newAcc.ActionTriggers[index].ThresholdType = strings.Replace(newAcc.ActionTriggers[index].ThresholdType, "_", "_event_", 1) - } - } - newAcc.InitCounters() - newAccounts = append(newAccounts, newAcc) - migratedKeys = append(migratedKeys, key) - } - // write data back - for _, newAcc := range newAccounts { - result, err := mig.ms.Marshal(newAcc) - if err != nil { - return err - } - if err := mig.db.Cmd("SET", utils.ACCOUNT_PREFIX+newAcc.ID, result).Err; err != nil { - return err - } - } - // delete old data - log.Printf("Deleting migrated accounts...") - for _, key := range migratedKeys { - if err := mig.db.Cmd("DEL", key).Err; err != nil { - return err - } - } - notMigrated := len(keys) - len(migratedKeys) - if notMigrated > 0 { - log.Printf("WARNING: there are %d accounts that failed migration!", notMigrated) - } - return err -} - -func (mig MigratorRC8) migrateActionTriggers() error { - keys, err := mig.db.Cmd("KEYS", utils.ACTION_TRIGGER_PREFIX+"*").List() - if err != nil { - return err - } - newAtrsMap := make(map[string]engine.ActionTriggers, len(keys)) - for _, key := range keys { - log.Printf("Migrating action trigger: %s...", key) - var oldAtrs ActionTriggers - var values []byte - if values, err = mig.db.Cmd("GET", key).Bytes(); err == nil { - if err := mig.ms.Unmarshal(values, &oldAtrs); err != nil { - return err - } - } - newAtrs := make(engine.ActionTriggers, len(oldAtrs)) - for index, oldAtr := range oldAtrs { - at := &engine.ActionTrigger{ - UniqueID: oldAtr.Id, - ThresholdType: oldAtr.ThresholdType, - ThresholdValue: oldAtr.ThresholdValue, - Recurrent: oldAtr.Recurrent, - MinSleep: oldAtr.MinSleep, - Weight: oldAtr.Weight, - ActionsID: oldAtr.ActionsId, - MinQueuedItems: oldAtr.MinQueuedItems, - Executed: oldAtr.Executed, - } - bf := &engine.BalanceFilter{} - if oldAtr.BalanceId != "" { - bf.ID = utils.StringPointer(oldAtr.BalanceId) - } - if oldAtr.BalanceType != "" { - bf.Type = utils.StringPointer(oldAtr.BalanceType) - } - if oldAtr.BalanceRatingSubject != "" { - bf.RatingSubject = utils.StringPointer(oldAtr.BalanceRatingSubject) - } - if oldAtr.BalanceDirection != "" { - bf.Directions = utils.StringMapPointer(utils.ParseStringMap(oldAtr.BalanceDirection)) - } - if oldAtr.BalanceDestinationIds != "" { - bf.DestinationIDs = utils.StringMapPointer(utils.ParseStringMap(oldAtr.BalanceDestinationIds)) - } - if oldAtr.BalanceTimingTags != "" { - bf.TimingIDs = utils.StringMapPointer(utils.ParseStringMap(oldAtr.BalanceTimingTags)) - } - if oldAtr.BalanceCategory != "" { - bf.Categories = utils.StringMapPointer(utils.ParseStringMap(oldAtr.BalanceCategory)) - } - if oldAtr.BalanceSharedGroup != "" { - bf.SharedGroups = utils.StringMapPointer(utils.ParseStringMap(oldAtr.BalanceSharedGroup)) - } - if oldAtr.BalanceWeight != 0 { - bf.Weight = utils.Float64Pointer(oldAtr.BalanceWeight) - } - if oldAtr.BalanceDisabled != false { - bf.Disabled = utils.BoolPointer(oldAtr.BalanceDisabled) - } - if !oldAtr.BalanceExpirationDate.IsZero() { - bf.ExpirationDate = utils.TimePointer(oldAtr.BalanceExpirationDate) - } - at.Balance = bf - newAtrs[index] = at - if newAtrs[index].ThresholdType == "*min_counter" || - newAtrs[index].ThresholdType == "*max_counter" { - newAtrs[index].ThresholdType = strings.Replace(newAtrs[index].ThresholdType, "_", "_event_", 1) - } - } - newAtrsMap[key] = newAtrs - } - // write data back - for key, atrs := range newAtrsMap { - result, err := mig.ms.Marshal(&atrs) - if err != nil { - return err - } - if err = mig.db.Cmd("SET", key, result).Err; err != nil { - return err - } - } - return nil -} - -func (mig MigratorRC8) migrateActions() error { - keys, err := mig.db.Cmd("KEYS", utils.ACTION_PREFIX+"*").List() - if err != nil { - return err - } - newAcsMap := make(map[string]engine.Actions, len(keys)) - for _, key := range keys { - log.Printf("Migrating action: %s...", key) - var oldAcs Actions - var values []byte - if values, err = mig.db.Cmd("GET", key).Bytes(); err == nil { - if err := mig.ms.Unmarshal(values, &oldAcs); err != nil { - return err - } - } - newAcs := make(engine.Actions, len(oldAcs)) - for index, oldAc := range oldAcs { - a := &engine.Action{ - Id: oldAc.Id, - ActionType: oldAc.ActionType, - ExtraParameters: oldAc.ExtraParameters, - ExpirationString: oldAc.ExpirationString, - Weight: oldAc.Weight, - Balance: &engine.BalanceFilter{}, - } - bf := a.Balance - if oldAc.Balance.Uuid != "" { - bf.Uuid = utils.StringPointer(oldAc.Balance.Uuid) - } - if oldAc.Balance.Id != "" { - bf.ID = utils.StringPointer(oldAc.Balance.Id) - } - if oldAc.BalanceType != "" { - bf.Type = utils.StringPointer(oldAc.BalanceType) - } - if oldAc.Balance.Value != 0 { - bf.Value = &utils.ValueFormula{Static: oldAc.Balance.Value} - } - if oldAc.Balance.RatingSubject != "" { - bf.RatingSubject = utils.StringPointer(oldAc.Balance.RatingSubject) - } - if oldAc.Balance.DestinationIds != "" { - bf.DestinationIDs = utils.StringMapPointer(utils.ParseStringMap(oldAc.Balance.DestinationIds)) - } - if oldAc.Balance.TimingIDs != "" { - bf.TimingIDs = utils.StringMapPointer(utils.ParseStringMap(oldAc.Balance.TimingIDs)) - } - if oldAc.Balance.Category != "" { - bf.Categories = utils.StringMapPointer(utils.ParseStringMap(oldAc.Balance.Category)) - } - if oldAc.Balance.SharedGroup != "" { - bf.SharedGroups = utils.StringMapPointer(utils.ParseStringMap(oldAc.Balance.SharedGroup)) - } - if oldAc.Balance.Weight != 0 { - bf.Weight = utils.Float64Pointer(oldAc.Balance.Weight) - } - if oldAc.Balance.Disabled != false { - bf.Disabled = utils.BoolPointer(oldAc.Balance.Disabled) - } - if !oldAc.Balance.ExpirationDate.IsZero() { - bf.ExpirationDate = utils.TimePointer(oldAc.Balance.ExpirationDate) - } - bf.Timings = oldAc.Balance.Timings - newAcs[index] = a - } - newAcsMap[key] = newAcs - } - // write data back - for key, acs := range newAcsMap { - result, err := mig.ms.Marshal(&acs) - if err != nil { - return err - } - if err = mig.db.Cmd("SET", key, result).Err; err != nil { - return err - } - } - return nil -} - -func (mig MigratorRC8) migrateDerivedChargers() error { - keys, err := mig.db.Cmd("KEYS", utils.DERIVEDCHARGERS_PREFIX+"*").List() - if err != nil { - return err - } - newDcsMap := make(map[string]*utils.DerivedChargers, len(keys)) - for _, key := range keys { - log.Printf("Migrating derived charger: %s...", key) - var oldDcs []*utils.DerivedCharger - var values []byte - if values, err = mig.db.Cmd("GET", key).Bytes(); err == nil { - if err := mig.ms.Unmarshal(values, &oldDcs); err != nil { - return err - } - } - newDcs := &utils.DerivedChargers{ - DestinationIDs: make(utils.StringMap), - Chargers: oldDcs, - } - newDcsMap[key] = newDcs - } - // write data back - for key, dcs := range newDcsMap { - result, err := mig.ms.Marshal(&dcs) - if err != nil { - return err - } - if err = mig.db.Cmd("SET", key, result).Err; err != nil { - return err - } - } - return nil -} - -func (mig MigratorRC8) migrateActionPlans() error { - keys, err := mig.db.Cmd("KEYS", utils.ACTION_PLAN_PREFIX+"*").List() - if err != nil { - return err - } - aplsMap := make(map[string]ActionPlans, len(keys)) - for _, key := range keys { - log.Printf("Migrating action plans: %s...", key) - var apls ActionPlans - var values []byte - if values, err = mig.db.Cmd("GET", key).Bytes(); err == nil { - if err := mig.ms.Unmarshal(values, &apls); err != nil { - return err - } - } - // change all AccountIds - for _, apl := range apls { - for idx, actionId := range apl.AccountIds { - // fix id - idElements := strings.Split(actionId, utils.CONCATENATED_KEY_SEP) - if len(idElements) != 3 { - //log.Printf("Malformed account ID %s", actionId) - continue - } - apl.AccountIds[idx] = fmt.Sprintf("%s:%s", idElements[1], idElements[2]) - } - } - aplsMap[key] = apls - } - // write data back - newAplMap := make(map[string]*engine.ActionPlan) - for key, apls := range aplsMap { - for _, apl := range apls { - newApl, exists := newAplMap[key] - if !exists { - newApl = &engine.ActionPlan{ - Id: apl.Id, - AccountIDs: make(utils.StringMap), - } - newAplMap[key] = newApl - } - if !apl.IsASAP() { - for _, accID := range apl.AccountIds { - if _, exists := newApl.AccountIDs[accID]; !exists { - newApl.AccountIDs[accID] = true - } - } - } - newApl.ActionTimings = append(newApl.ActionTimings, &engine.ActionTiming{ - Uuid: utils.GenUUID(), - Timing: apl.Timing, - ActionsID: apl.ActionsId, - Weight: apl.Weight, - }) - } - } - for key, apl := range newAplMap { - result, err := mig.ms.Marshal(apl) - if err != nil { - return err - } - var b bytes.Buffer - w := zlib.NewWriter(&b) - w.Write(result) - w.Close() - if err = mig.db.Cmd("SET", key, b.Bytes()).Err; err != nil { - return err - } - } - return nil -} - -func (mig MigratorRC8) migrateSharedGroups() error { - keys, err := mig.db.Cmd("KEYS", utils.SHARED_GROUP_PREFIX+"*").List() - if err != nil { - return err - } - newShgMap := make(map[string]*engine.SharedGroup, len(keys)) - for _, key := range keys { - log.Printf("Migrating shared groups: %s...", key) - oldShg := SharedGroup{} - var values []byte - if values, err = mig.db.Cmd("GET", key).Bytes(); err == nil { - if err := mig.ms.Unmarshal(values, &oldShg); err != nil { - return err - } - } - newShg := &engine.SharedGroup{ - Id: oldShg.Id, - AccountParameters: oldShg.AccountParameters, - MemberIds: make(utils.StringMap), - } - for _, accID := range oldShg.MemberIds { - newShg.MemberIds[accID] = true - } - newShgMap[key] = newShg - } - // write data back - for key, shg := range newShgMap { - result, err := mig.ms.Marshal(&shg) - if err != nil { - return err - } - if err = mig.db.Cmd("SET", key, result).Err; err != nil { - return err - } - } - return nil -} - -func (mig MigratorRC8) writeVersion() error { - result, err := mig.ms.Marshal(engine.CurrentVersion) - if err != nil { - return err - } - return mig.db.Cmd("SET", utils.VERSION_PREFIX+"struct", result).Err -} diff --git a/cmd/cgr-loader/migrator_rc8int.go b/cmd/cgr-loader/migrator_rc8int.go deleted file mode 100644 index 0da38821d..000000000 --- a/cmd/cgr-loader/migrator_rc8int.go +++ /dev/null @@ -1,426 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package main - -import ( - "log" - "time" - - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -type Account1 struct { - Id string - BalanceMap map[string]BalanceChain1 - UnitCounters UnitCounters1 - ActionTriggers ActionTriggers1 - AllowNegative bool - Disabled bool -} - -type Balance1 struct { - Uuid string //system wide unique - Id string // account wide unique - Value float64 - Directions utils.StringMap - ExpirationDate time.Time - Weight float64 - DestinationIds utils.StringMap - RatingSubject string - Categories utils.StringMap - SharedGroups utils.StringMap - Timings []*engine.RITiming - TimingIDs utils.StringMap - Disabled bool - Factor engine.ValueFactor - Blocker bool - precision int - account *Account // used to store ub reference for shared balances - dirty bool -} - -type BalanceChain1 []*Balance1 - -type UnitCounter1 struct { - BalanceType string // *monetary/*voice/*sms/etc - CounterType string // *event or *balance - Balances BalanceChain1 // first balance is the general one (no destination) -} - -type UnitCounters1 []*UnitCounter1 - -type Action1 struct { - Id string - ActionType string - BalanceType string - ExtraParameters string - Filter string - ExpirationString string // must stay as string because it can have relative values like 1month - Weight float64 - Balance *Balance1 -} - -type Actions1 []*Action1 - -type ActionTrigger1 struct { - ID string // original csv tag - UniqueID string // individual id - ThresholdType string - ThresholdValue float64 - Recurrent bool // reset eexcuted flag each run - MinSleep time.Duration // Minimum duration between two executions in case of recurrent triggers - BalanceId string - BalanceType string // *monetary/*voice etc - BalanceDirections utils.StringMap // filter for balance - BalanceDestinationIds utils.StringMap // filter for balance - BalanceWeight float64 // filter for balance - BalanceExpirationDate time.Time // filter for balance - BalanceTimingTags utils.StringMap // filter for balance - BalanceRatingSubject string // filter for balance - BalanceCategories utils.StringMap // filter for balance - BalanceSharedGroups utils.StringMap // filter for balance - BalanceBlocker bool - BalanceDisabled bool // filter for balance - Weight float64 - ActionsId string - MinQueuedItems int // Trigger actions only if this number is hit (stats only) - Executed bool -} -type ActionTriggers1 []*ActionTrigger1 - -func (mig MigratorRC8) migrateAccountsInt() error { - keys, err := mig.db.Cmd("KEYS", utils.ACCOUNT_PREFIX+"*").List() - if err != nil { - return err - } - newAccounts := make([]*engine.Account, 0) - var migratedKeys []string - // get existing accounts - for _, key := range keys { - log.Printf("Migrating account: %s...", key) - values, err := mig.db.Cmd("GET", key).Bytes() - if err != nil { - continue - } - var oldAcc Account1 - if err = mig.ms.Unmarshal(values, &oldAcc); err != nil { - return err - } - // transfer data into new structurse - newAcc := &engine.Account{ - ID: oldAcc.Id, - BalanceMap: make(map[string]engine.Balances, len(oldAcc.BalanceMap)), - UnitCounters: make(engine.UnitCounters), - ActionTriggers: make(engine.ActionTriggers, len(oldAcc.ActionTriggers)), - AllowNegative: oldAcc.AllowNegative, - Disabled: oldAcc.Disabled, - } - // balances - balanceErr := false - for key, oldBalChain := range oldAcc.BalanceMap { - newAcc.BalanceMap[key] = make(engine.Balances, len(oldBalChain)) - for index, oldBal := range oldBalChain { - newAcc.BalanceMap[key][index] = &engine.Balance{ - Uuid: oldBal.Uuid, - ID: oldBal.Id, - Value: oldBal.Value, - Directions: oldBal.Directions, - ExpirationDate: oldBal.ExpirationDate, - Weight: oldBal.Weight, - DestinationIDs: oldBal.DestinationIds, - RatingSubject: oldBal.RatingSubject, - Categories: oldBal.Categories, - SharedGroups: oldBal.SharedGroups, - Timings: oldBal.Timings, - TimingIDs: oldBal.TimingIDs, - Disabled: oldBal.Disabled, - Factor: oldBal.Factor, - Blocker: oldBal.Blocker, - } - } - } - if balanceErr { - continue - } - // unit counters - for _, oldUc := range oldAcc.UnitCounters { - newUc := &engine.UnitCounter{ - Counters: make(engine.CounterFilters, len(oldUc.Balances)), - } - for index, oldUcBal := range oldUc.Balances { - b := &engine.Balance{ - Uuid: oldUcBal.Uuid, - ID: oldUcBal.Id, - Value: oldUcBal.Value, - Directions: oldUcBal.Directions, - ExpirationDate: oldUcBal.ExpirationDate, - Weight: oldUcBal.Weight, - DestinationIDs: oldUcBal.DestinationIds, - RatingSubject: oldUcBal.RatingSubject, - Categories: oldUcBal.Categories, - SharedGroups: oldUcBal.SharedGroups, - Timings: oldUcBal.Timings, - TimingIDs: oldUcBal.TimingIDs, - Disabled: oldUcBal.Disabled, - Factor: oldUcBal.Factor, - Blocker: oldUcBal.Blocker, - } - bf := &engine.BalanceFilter{} - bf.LoadFromBalance(b) - cf := &engine.CounterFilter{ - Value: oldUcBal.Value, - Filter: bf, - } - newUc.Counters[index] = cf - } - newAcc.UnitCounters[oldUc.BalanceType] = append(newAcc.UnitCounters[oldUc.BalanceType], newUc) - } - // action triggers - for index, oldAtr := range oldAcc.ActionTriggers { - at := &engine.ActionTrigger{ - ID: oldAtr.ID, - UniqueID: oldAtr.UniqueID, - ThresholdType: oldAtr.ThresholdType, - ThresholdValue: oldAtr.ThresholdValue, - Recurrent: oldAtr.Recurrent, - MinSleep: oldAtr.MinSleep, - Weight: oldAtr.Weight, - ActionsID: oldAtr.ActionsId, - MinQueuedItems: oldAtr.MinQueuedItems, - Executed: oldAtr.Executed, - } - bf := &engine.BalanceFilter{} - if oldAtr.BalanceId != "" { - bf.ID = utils.StringPointer(oldAtr.BalanceId) - } - if oldAtr.BalanceType != "" { - bf.Type = utils.StringPointer(oldAtr.BalanceType) - } - if oldAtr.BalanceRatingSubject != "" { - bf.RatingSubject = utils.StringPointer(oldAtr.BalanceRatingSubject) - } - if !oldAtr.BalanceDirections.IsEmpty() { - bf.Directions = utils.StringMapPointer(oldAtr.BalanceDirections) - } - if !oldAtr.BalanceDestinationIds.IsEmpty() { - bf.DestinationIDs = utils.StringMapPointer(oldAtr.BalanceDestinationIds) - } - if !oldAtr.BalanceTimingTags.IsEmpty() { - bf.TimingIDs = utils.StringMapPointer(oldAtr.BalanceTimingTags) - } - if !oldAtr.BalanceCategories.IsEmpty() { - bf.Categories = utils.StringMapPointer(oldAtr.BalanceCategories) - } - if !oldAtr.BalanceSharedGroups.IsEmpty() { - bf.SharedGroups = utils.StringMapPointer(oldAtr.BalanceSharedGroups) - } - if oldAtr.BalanceWeight != 0 { - bf.Weight = utils.Float64Pointer(oldAtr.BalanceWeight) - } - if oldAtr.BalanceDisabled != false { - bf.Disabled = utils.BoolPointer(oldAtr.BalanceDisabled) - } - if !oldAtr.BalanceExpirationDate.IsZero() { - bf.ExpirationDate = utils.TimePointer(oldAtr.BalanceExpirationDate) - } - at.Balance = bf - newAcc.ActionTriggers[index] = at - } - newAcc.InitCounters() - newAccounts = append(newAccounts, newAcc) - migratedKeys = append(migratedKeys, key) - } - // write data back - for _, newAcc := range newAccounts { - result, err := mig.ms.Marshal(newAcc) - if err != nil { - return err - } - if err := mig.db.Cmd("SET", utils.ACCOUNT_PREFIX+newAcc.ID, result).Err; err != nil { - return err - } - } - notMigrated := len(keys) - len(migratedKeys) - if notMigrated > 0 { - log.Printf("WARNING: there are %d accounts that failed migration!", notMigrated) - } - return err -} - -func (mig MigratorRC8) migrateActionTriggersInt() error { - keys, err := mig.db.Cmd("KEYS", utils.ACTION_TRIGGER_PREFIX+"*").List() - if err != nil { - return err - } - newAtrsMap := make(map[string]engine.ActionTriggers, len(keys)) - for _, key := range keys { - log.Printf("Migrating action trigger: %s...", key) - var oldAtrs ActionTriggers1 - var values []byte - if values, err = mig.db.Cmd("GET", key).Bytes(); err == nil { - if err := mig.ms.Unmarshal(values, &oldAtrs); err != nil { - return err - } - } - newAtrs := make(engine.ActionTriggers, len(oldAtrs)) - for index, oldAtr := range oldAtrs { - at := &engine.ActionTrigger{ - ID: oldAtr.ID, - UniqueID: oldAtr.UniqueID, - ThresholdType: oldAtr.ThresholdType, - ThresholdValue: oldAtr.ThresholdValue, - Recurrent: oldAtr.Recurrent, - MinSleep: oldAtr.MinSleep, - Weight: oldAtr.Weight, - ActionsID: oldAtr.ActionsId, - MinQueuedItems: oldAtr.MinQueuedItems, - Executed: oldAtr.Executed, - } - bf := &engine.BalanceFilter{} - if oldAtr.BalanceId != "" { - bf.ID = utils.StringPointer(oldAtr.BalanceId) - } - if oldAtr.BalanceType != "" { - bf.Type = utils.StringPointer(oldAtr.BalanceType) - } - if oldAtr.BalanceRatingSubject != "" { - bf.RatingSubject = utils.StringPointer(oldAtr.BalanceRatingSubject) - } - if !oldAtr.BalanceDirections.IsEmpty() { - bf.Directions = utils.StringMapPointer(oldAtr.BalanceDirections) - } - if !oldAtr.BalanceDestinationIds.IsEmpty() { - bf.DestinationIDs = utils.StringMapPointer(oldAtr.BalanceDestinationIds) - } - if !oldAtr.BalanceTimingTags.IsEmpty() { - bf.TimingIDs = utils.StringMapPointer(oldAtr.BalanceTimingTags) - } - if !oldAtr.BalanceCategories.IsEmpty() { - bf.Categories = utils.StringMapPointer(oldAtr.BalanceCategories) - } - if !oldAtr.BalanceSharedGroups.IsEmpty() { - bf.SharedGroups = utils.StringMapPointer(oldAtr.BalanceSharedGroups) - } - if oldAtr.BalanceWeight != 0 { - bf.Weight = utils.Float64Pointer(oldAtr.BalanceWeight) - } - if oldAtr.BalanceDisabled != false { - bf.Disabled = utils.BoolPointer(oldAtr.BalanceDisabled) - } - if !oldAtr.BalanceExpirationDate.IsZero() { - bf.ExpirationDate = utils.TimePointer(oldAtr.BalanceExpirationDate) - } - at.Balance = bf - newAtrs[index] = at - } - newAtrsMap[key] = newAtrs - } - // write data back - for key, atrs := range newAtrsMap { - result, err := mig.ms.Marshal(&atrs) - if err != nil { - return err - } - if err = mig.db.Cmd("SET", key, result).Err; err != nil { - return err - } - } - return nil -} - -func (mig MigratorRC8) migrateActionsInt() error { - keys, err := mig.db.Cmd("KEYS", utils.ACTION_PREFIX+"*").List() - if err != nil { - return err - } - newAcsMap := make(map[string]engine.Actions, len(keys)) - for _, key := range keys { - log.Printf("Migrating action: %s...", key) - var oldAcs Actions1 - var values []byte - if values, err = mig.db.Cmd("GET", key).Bytes(); err == nil { - if err := mig.ms.Unmarshal(values, &oldAcs); err != nil { - return err - } - } - newAcs := make(engine.Actions, len(oldAcs)) - for index, oldAc := range oldAcs { - a := &engine.Action{ - Id: oldAc.Id, - ActionType: oldAc.ActionType, - ExtraParameters: oldAc.ExtraParameters, - ExpirationString: oldAc.ExpirationString, - Filter: oldAc.Filter, - Weight: oldAc.Weight, - Balance: &engine.BalanceFilter{}, - } - bf := a.Balance - if oldAc.Balance.Uuid != "" { - bf.Uuid = utils.StringPointer(oldAc.Balance.Uuid) - } - if oldAc.Balance.Id != "" { - bf.ID = utils.StringPointer(oldAc.Balance.Id) - } - if oldAc.BalanceType != "" { - bf.Type = utils.StringPointer(oldAc.BalanceType) - } - if oldAc.Balance.Value != 0 { - bf.Value = &utils.ValueFormula{Static: oldAc.Balance.Value} - } - if oldAc.Balance.RatingSubject != "" { - bf.RatingSubject = utils.StringPointer(oldAc.Balance.RatingSubject) - } - if !oldAc.Balance.DestinationIds.IsEmpty() { - bf.DestinationIDs = utils.StringMapPointer(oldAc.Balance.DestinationIds) - } - if !oldAc.Balance.TimingIDs.IsEmpty() { - bf.TimingIDs = utils.StringMapPointer(oldAc.Balance.TimingIDs) - } - if !oldAc.Balance.Categories.IsEmpty() { - bf.Categories = utils.StringMapPointer(oldAc.Balance.Categories) - } - if !oldAc.Balance.SharedGroups.IsEmpty() { - bf.SharedGroups = utils.StringMapPointer(oldAc.Balance.SharedGroups) - } - if oldAc.Balance.Weight != 0 { - bf.Weight = utils.Float64Pointer(oldAc.Balance.Weight) - } - if oldAc.Balance.Disabled != false { - bf.Disabled = utils.BoolPointer(oldAc.Balance.Disabled) - } - if !oldAc.Balance.ExpirationDate.IsZero() { - bf.ExpirationDate = utils.TimePointer(oldAc.Balance.ExpirationDate) - } - bf.Timings = oldAc.Balance.Timings - newAcs[index] = a - } - newAcsMap[key] = newAcs - } - // write data back - for key, acs := range newAcsMap { - result, err := mig.ms.Marshal(&acs) - if err != nil { - return err - } - if err = mig.db.Cmd("SET", key, result).Err; err != nil { - return err - } - } - return nil -} diff --git a/cmd/cgr-loader/migrator_rc8int2.go b/cmd/cgr-loader/migrator_rc8int2.go deleted file mode 100644 index f72e58e80..000000000 --- a/cmd/cgr-loader/migrator_rc8int2.go +++ /dev/null @@ -1,119 +0,0 @@ -/* -Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments -Copyright (C) ITsysCOM GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see -*/ -package main - -import ( - "log" - "time" - - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" -) - -type BalanceFilter2 struct { - Uuid *string - ID *string - Type *string - Value *float64 - Directions *utils.StringMap - ExpirationDate *time.Time - Weight *float64 - DestinationIDs *utils.StringMap - RatingSubject *string - Categories *utils.StringMap - SharedGroups *utils.StringMap - TimingIDs *utils.StringMap - Timings []*engine.RITiming - Disabled *bool - Factor *engine.ValueFactor - Blocker *bool -} - -type Action2 struct { - Id string - ActionType string - ExtraParameters string - Filter string - ExpirationString string // must stay as string because it can have relative values like 1month - Weight float64 - Balance *BalanceFilter2 -} - -type Actions2 []*Action2 - -func (mig MigratorRC8) migrateActionsInt2() error { - keys, err := mig.db.Cmd("KEYS", utils.ACTION_PREFIX+"*").List() - if err != nil { - return err - } - newAcsMap := make(map[string]engine.Actions, len(keys)) - for _, key := range keys { - log.Printf("Migrating action: %s...", key) - var oldAcs Actions2 - var values []byte - if values, err = mig.db.Cmd("GET", key).Bytes(); err == nil { - if err := mig.ms.Unmarshal(values, &oldAcs); err != nil { - return err - } - } - newAcs := make(engine.Actions, len(oldAcs)) - for index, oldAc := range oldAcs { - a := &engine.Action{ - Id: oldAc.Id, - ActionType: oldAc.ActionType, - ExtraParameters: oldAc.ExtraParameters, - ExpirationString: oldAc.ExpirationString, - Filter: oldAc.Filter, - Weight: oldAc.Weight, - Balance: &engine.BalanceFilter{ - Uuid: oldAc.Balance.Uuid, - ID: oldAc.Balance.ID, - Type: oldAc.Balance.Type, - Directions: oldAc.Balance.Directions, - ExpirationDate: oldAc.Balance.ExpirationDate, - Weight: oldAc.Balance.Weight, - DestinationIDs: oldAc.Balance.DestinationIDs, - RatingSubject: oldAc.Balance.RatingSubject, - Categories: oldAc.Balance.Categories, - SharedGroups: oldAc.Balance.SharedGroups, - TimingIDs: oldAc.Balance.TimingIDs, - Timings: oldAc.Balance.Timings, - Disabled: oldAc.Balance.Disabled, - Factor: oldAc.Balance.Factor, - Blocker: oldAc.Balance.Blocker, - }, - } - if oldAc.Balance.Value != nil { - a.Balance.Value = &utils.ValueFormula{Static: *oldAc.Balance.Value} - } - newAcs[index] = a - } - newAcsMap[key] = newAcs - } - // write data back - for key, acs := range newAcsMap { - result, err := mig.ms.Marshal(&acs) - if err != nil { - return err - } - if err = mig.db.Cmd("SET", key, result).Err; err != nil { - return err - } - } - return nil -} diff --git a/cmd/cgr-migrator/cgr-migrator.go b/cmd/cgr-migrator/cgr-migrator.go index 8fb5682f0..2e056091e 100755 --- a/cmd/cgr-migrator/cgr-migrator.go +++ b/cmd/cgr-migrator/cgr-migrator.go @@ -29,6 +29,11 @@ import ( ) var ( + oldDataDB migrator.V1DataDB + oldstorDB engine.Storage + oStorDBType string + odataDBType string + oDBDataEncoding string migrate = flag.String("migrate", "", "Fire up automatic migration <*set_versions|*cost_details|*accounts|*actions|*action_triggers|*action_plans|*shared_groups>") version = flag.Bool("version", false, "Prints the application version.") @@ -46,14 +51,14 @@ var ( storDBUser = flag.String("stordb_user", config.CgrConfig().StorDBUser, "The storDb user to sign in as.") storDBPass = flag.String("stordb_passwd", config.CgrConfig().StorDBPass, "The storDb user's password.") - oldDataDBType = flag.String("old_datadb_type", config.CgrConfig().DataDbType, "The type of the DataDb database ") + oldDataDBType = flag.String("old_datadb_type", "", "The type of the DataDb database ") oldDataDBHost = flag.String("old_datadb_host", config.CgrConfig().DataDbHost, "The DataDb host to connect to.") oldDataDBPort = flag.String("old_datadb_port", config.CgrConfig().DataDbPort, "The DataDb port to bind to.") - oldDataDBName = flag.String("old_datadb_name", "11", "The name/number of the DataDb to connect to.") + oldDataDBName = flag.String("old_datadb_name", config.CgrConfig().DataDbName, "The name/number of the DataDb to connect to.") oldDataDBUser = flag.String("old_datadb_user", config.CgrConfig().DataDbUser, "The DataDb user to sign in as.") oldDataDBPass = flag.String("old_datadb_passwd", config.CgrConfig().DataDbPass, "The DataDb user's password.") - oldStorDBType = flag.String("old_stordb_type", config.CgrConfig().StorDBType, "The type of the storDb database ") + oldStorDBType = flag.String("old_stordb_type", "", "The type of the storDb database ") oldStorDBHost = flag.String("old_stordb_host", config.CgrConfig().StorDBHost, "The storDb host to connect to.") oldStorDBPort = flag.String("old_stordb_port", config.CgrConfig().StorDBPort, "The storDb port to bind to.") oldStorDBName = flag.String("old_stordb_name", config.CgrConfig().StorDBName, "The name/number of the storDb to connect to.") @@ -61,10 +66,10 @@ var ( oldStorDBPass = flag.String("old_stordb_passwd", config.CgrConfig().StorDBPass, "The storDb user's password.") loadHistorySize = flag.Int("load_history_size", config.CgrConfig().LoadHistorySize, "Limit the number of records in the load history") - oldLoadHistorySize = flag.Int("old_load_history_size", config.CgrConfig().LoadHistorySize, "Limit the number of records in the load history") + oldLoadHistorySize = flag.Int("old_load_history_size", 0, "Limit the number of records in the load history") dbDataEncoding = flag.String("dbdata_encoding", config.CgrConfig().DBDataEncoding, "The encoding used to store object data in strings") - oldDBDataEncoding = flag.String("old_dbdata_encoding", config.CgrConfig().DBDataEncoding, "The encoding used to store object data in strings") + oldDBDataEncoding = flag.String("old_dbdata_encoding", "", "The encoding used to store object data in strings") //nu salvez doar citesc din oldDb //dryRun = flag.Bool("dry_run", false, "When true will not save loaded data to dataDb but just parse it for consistency and errors.") //verbose = flag.Bool("verbose", false, "Enable detailed verbose logging output") @@ -80,26 +85,38 @@ func main() { return } if migrate != nil && *migrate != "" { // Run migrator - dataDB, err := engine.ConfigureDataStorage(*dataDBType, *dataDBHost, *dataDBPort, *dataDBName, *dataDBUser, *dataDBPass, *dbDataEncoding, config.CgrConfig().CacheConfig, *loadHistorySize) if err != nil { log.Fatal(err) } - oldDataDB, err := migrator.ConfigureV1DataStorage(*oldDataDBType, *oldDataDBHost, *oldDataDBPort, *oldDataDBName, *oldDataDBUser, *oldDataDBPass, *oldDBDataEncoding) + storDB, err := engine.ConfigureStorStorage(*storDBType, *storDBHost, *storDBPort, *storDBName, *storDBUser, *storDBPass, *dbDataEncoding, + config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes) if err != nil { log.Fatal(err) } - storDB, err := engine.ConfigureStorStorage(*storDBType, *storDBHost, *storDBPort, *storDBName, *storDBUser, *storDBPass, *dbDataEncoding, + + if *oldDataDBType==""{ + *oldDataDBType=*dataDBType + *oldDataDBHost=*dataDBHost + *oldDataDBPort=*dataDBPort + *oldDataDBName=*dataDBName + *oldDataDBUser=*dataDBUser + *oldDataDBPass=*dataDBPass + + } + oldDataDB, err := migrator.ConfigureV1DataStorage(*oldDataDBType, *oldDataDBHost, *dataDBPort, *dataDBName, *dataDBUser, *dataDBPass, *dbDataEncoding) + if err != nil { + log.Fatal(err) + } + oldstorDB=storDB + if *oldStorDBType!="" { + oldstorDB, err = engine.ConfigureStorStorage(oStorDBType, *oldStorDBHost, *oldStorDBPort, *oldStorDBName, *oldStorDBUser, *oldStorDBPass, *oldDBDataEncoding, config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes) if err != nil { log.Fatal(err) + } } - oldstorDB, err := engine.ConfigureStorStorage(*oldStorDBType, *oldStorDBHost, *oldStorDBPort, *oldStorDBName, *oldStorDBUser, *oldStorDBPass, *oldDBDataEncoding, - config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes) - if err != nil { - log.Fatal(err) - } - m,err := migrator.NewMigrator(dataDB, *dataDBType, *dbDataEncoding, storDB, *storDBType,oldDataDB,*oldDataDBType,*oldDBDataEncoding,oldstorDB,*oldStorDBType) + m,err := migrator.NewMigrator(dataDB, *dataDBType, *dbDataEncoding, storDB, *storDBType, oldDataDB, *oldDataDBType,*oldDBDataEncoding,oldstorDB,*oldStorDBType) if err != nil { log.Fatal(err) } diff --git a/engine/libtest.go b/engine/libtest.go index 1a8e55dfc..6d7678670 100644 --- a/engine/libtest.go +++ b/engine/libtest.go @@ -41,7 +41,10 @@ func InitDataDb(cfg *config.CGRConfig) error { return err } dataDB.LoadRatingCache(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) - CheckVersions(dataDB) // Write version before starting + // Write version before starting + if err := CheckVersions(dataDB); err != nil { + return err + } return nil } diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 436233cac..5b6cf617c 100755 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -38,6 +38,7 @@ type Storage interface { SetVersions(vrs Versions, overwrite bool) (err error) RemoveVersions(vrs Versions) (err error) SelectDatabase(dbName string) (err error) + GetStorageType() string } // OnlineStorage contains methods to use for administering online data diff --git a/engine/storage_map.go b/engine/storage_map.go index 22925427e..50bc5b4cf 100755 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -1288,30 +1288,6 @@ func (ms *MapStorage) GetSMCost(cgrid, source, runid, originHost, originID strin return } -func (ms *MapStorage) SetStructVersion(v *StructVersion) (err error) { - ms.mu.Lock() - defer ms.mu.Unlock() - var result []byte - result, err = ms.ms.Marshal(v) - if err != nil { - return - } - ms.dict[utils.VERSION_PREFIX+"struct"] = result - return -} - -func (ms *MapStorage) GetStructVersion() (rsv *StructVersion, err error) { - ms.mu.RLock() - defer ms.mu.RUnlock() - rsv = &StructVersion{} - if values, ok := ms.dict[utils.VERSION_PREFIX+"struct"]; ok { - err = ms.ms.Unmarshal(values, &rsv) - } else { - return nil, utils.ErrNotFound - } - return -} - func (ms *MapStorage) GetResourceProfile(id string, skipCache bool, transactionID string) (rsp *ResourceProfile, err error) { ms.mu.RLock() defer ms.mu.RUnlock() @@ -1510,18 +1486,6 @@ func (ms *MapStorage) MatchReqFilterIndex(dbKey, fldName, fldVal string) (itemID return } -func (ms *MapStorage) GetVersions(itm string) (vrs Versions, err error) { - return -} - -func (ms *MapStorage) SetVersions(vrs Versions, overwrite bool) (err error) { - return -} - -func (ms *MapStorage) RemoveVersions(vrs Versions) (err error) { - return -} - // GetStatsQueue retrieves a StatsQueue from dataDB func (ms *MapStorage) GetStatsConfig(sqID string) (scf *StatsConfig, err error) { ms.mu.RLock() @@ -1650,7 +1614,6 @@ func (ms *MapStorage) RemThresholdCfg(sqID string, transactionID string) (err er return } - func (ms *MapStorage) GetVersions(itm string) (vrs Versions, err error) { ms.mu.Lock() defer ms.mu.Unlock() @@ -1697,4 +1660,15 @@ func (ms *MapStorage) SetVersions(vrs Versions, overwrite bool) (err error) { ms.dict[utils.TBLVersions] = result return } -} \ No newline at end of file +} + +func (ms *MapStorage) RemoveVersions(vrs Versions) (err error) { + ms.mu.Lock() + defer ms.mu.Unlock() + delete(ms.dict, utils.TBLVersions) + return +} + +func (ms *MapStorage) GetStorageType() string { + return utils.MAPSTOR +} diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 254562162..e12a78fd9 100755 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -2134,4 +2134,4 @@ func (ms *MongoStorage) RemThresholdCfg(ID string, transactionID string) (err er cache.RemKey(key, cacheCommit(transactionID), transactionID) session.Close() return -} \ No newline at end of file +} diff --git a/engine/storage_mongo_stordb.go b/engine/storage_mongo_stordb.go index 735402144..9a0088a4f 100755 --- a/engine/storage_mongo_stordb.go +++ b/engine/storage_mongo_stordb.go @@ -1237,3 +1237,7 @@ func (ms *MongoStorage) RemoveVersions(vrs Versions) (err error) { } return nil } + +func (ms *MongoStorage) GetStorageType() string { + return utils.MONGO +} diff --git a/engine/storage_mysql.go b/engine/storage_mysql.go index 7681d6b7e..517a1fab6 100644 --- a/engine/storage_mysql.go +++ b/engine/storage_mysql.go @@ -84,3 +84,7 @@ func (self *MySQLStorage) notExtraFieldsExistsQry(field string) string { func (self *MySQLStorage) notExtraFieldsValueQry(field, value string) string { return fmt.Sprintf(" extra_fields NOT LIKE '%%\"%s\":\"%s\"%%'", field, value) } + +func (self *SQLStorage) GetStorageType() string { + return utils.MYSQL +} diff --git a/engine/storage_postgres.go b/engine/storage_postgres.go index 20e1a06a7..0dc6e5391 100644 --- a/engine/storage_postgres.go +++ b/engine/storage_postgres.go @@ -88,3 +88,7 @@ func (self *PostgresStorage) notExtraFieldsExistsQry(field string) string { func (self *PostgresStorage) notExtraFieldsValueQry(field, value string) string { return fmt.Sprintf(" NOT (extra_fields ?'%s' AND (extra_fields ->> '%s') = '%s')", field, field, value) } + +func (self *PostgresStorage) GetStorageType() string { + return utils.POSTGRES +} diff --git a/engine/storage_redis.go b/engine/storage_redis.go index e8e26cfa2..280926c2c 100755 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -1357,27 +1357,6 @@ func (rs *RedisStorage) GetAllCdrStats() (css []*CdrStats, err error) { return } -func (rs *RedisStorage) SetStructVersion(v *StructVersion) (err error) { - var result []byte - result, err = rs.ms.Marshal(v) - if err != nil { - return - } - return rs.Cmd("SET", utils.VERSION_PREFIX+"struct", result).Err -} - -func (rs *RedisStorage) GetStructVersion() (rsv *StructVersion, err error) { - var values []byte - if values, err = rs.Cmd("GET", utils.VERSION_PREFIX+"struct").Bytes(); err != nil { - if err == redis.ErrRespNil { // did not find the destination - err = utils.ErrNotFound - } - return - } - err = rs.ms.Unmarshal(values, &rsv) - return -} - func (rs *RedisStorage) GetResourceProfile(id string, skipCache bool, transactionID string) (rsp *ResourceProfile, err error) { key := utils.ResourceProfilesPrefix + id if !skipCache { @@ -1734,3 +1713,7 @@ func (rs *RedisStorage) RemThresholdCfg(ID string, transactionID string) (err er cache.RemKey(key, cacheCommit(transactionID), transactionID) return } + +func (rs *RedisStorage) GetStorageType() string { + return utils.REDIS +} diff --git a/engine/storage_utils.go b/engine/storage_utils.go index 1e64c6e19..efddd7ec6 100755 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -58,19 +58,6 @@ func ConfigureDataStorage(db_type, host, port, name, user, pass, marshaler strin func ConfigureStorStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn, connMaxLifetime int, cdrsIndexes []string) (db Storage, err error) { var d Storage switch db_type { - /* - case utils.REDIS: - var db_nb int - db_nb, err = strconv.Atoi(name) - if err != nil { - utils.Logger.Crit("Redis db name must be an integer!") - return nil, err - } - if port != "" { - host += ":" + port - } - d, err = NewRedisStorage(host, db_nb, pass, marshaler) - */ case utils.MONGO: d, err = NewMongoStorage(host, port, name, user, pass, utils.StorDB, cdrsIndexes, nil, 1) case utils.POSTGRES: diff --git a/engine/version.go b/engine/version.go index 3be7f0844..cf1a4aa6b 100644 --- a/engine/version.go +++ b/engine/version.go @@ -20,17 +20,20 @@ package engine import ( "errors" "fmt" - "github.com/cgrates/cgrates/utils" + "log" ) func CheckVersions(storage Storage) error { - x := Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} // get current db version if storage == nil { storage = dataStorage } + storType := storage.GetStorageType() + log.Print(storType) + x := CurrentDBVersions(storType) dbVersion, err := storage.GetVersions(utils.TBLVersions) + log.Print(dbVersion) if err != nil { // no data, write version if err := storage.SetVersions(x, false); err != nil { @@ -39,20 +42,19 @@ func CheckVersions(storage Storage) error { } else { // comparing versions - message := dbVersion.Compare(x) - if len(message) > 0 { + message := dbVersion.Compare(x, storType) + if len(message) > 2 { // write the new values msg := "Migration needed: please backup cgr data and run : <" + message + ">" - utils.Logger.Crit(msg) return errors.New(msg) } } return nil } -<<<<<<< HEAD -func (vers Versions) Compare(curent Versions) string { - x := map[string]string{ +func (vers Versions) Compare(curent Versions, storType string) string { + var x map[string]string + m := map[string]string{ utils.Accounts: "cgr-migrator -migrate=*accounts", utils.Actions: "cgr-migrator -migrate=*actions", utils.ActionTriggers: "cgr-migrator -migrate=*action_triggers", @@ -60,200 +62,56 @@ func (vers Versions) Compare(curent Versions) string { utils.SharedGroups: "cgr-migrator -migrate=*shared_groups", utils.COST_DETAILS: "cgr-migrator -migrate=*cost_details", } - for x, val := range x { - if vers[x] != curent[x] { + data := map[string]string{ + utils.Accounts: "cgr-migrator -migrate=*accounts", + utils.Actions: "cgr-migrator -migrate=*actions", + utils.ActionTriggers: "cgr-migrator -migrate=*action_triggers", + utils.ActionPlans: "cgr-migrator -migrate=*action_plans", + utils.SharedGroups: "cgr-migrator -migrate=*shared_groups", + } + stor := map[string]string{ + utils.COST_DETAILS: "cgr-migrator -migrate=*cost_details", + } + switch storType { + case utils.MONGO: + x = m + case utils.POSTGRES: + x = stor + case utils.MYSQL: + x = stor + case utils.REDIS: + x = data + case utils.MAPSTOR: + x = m + } + for y, val := range x { + log.Print("x:", vers[y], "y", curent[y]) + + if vers[y] != curent[y] { return val } } return "" } +func CurrentDBVersions(storType string) Versions { + switch storType { + case utils.MONGO: + return Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} + case utils.POSTGRES: + return Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2} + case utils.MYSQL: + return Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2} + case utils.REDIS: + return Versions{utils.COST_DETAILS: 2} + case utils.MAPSTOR: + return Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} + } + return nil +} + func CurrentStorDBVersions() Versions { return Versions{utils.COST_DETAILS: 2} -======= -var ( - CurrentVersion = &StructVersion{ - Destinations: "1", - RatingPlans: "1", - RatingProfiles: "1", - Lcrs: "1", - DerivedChargers: "1", - Actions: "1", - ActionPlans: "1", - ActionTriggers: "1", - SharedGroups: "1", - Accounts: "1", - CdrStats: "1", - Users: "1", - Alias: "1", - PubSubs: "1", - LoadHistory: "1", - Cdrs: "1", - SMCosts: "1", - ResourceProfiles: "1", - Timings: "1", - } -) - -type StructVersion struct { - Destinations string - RatingPlans string - RatingProfiles string - Lcrs string - DerivedChargers string - Actions string - ActionPlans string - ActionTriggers string - SharedGroups string - Accounts string - CdrStats string - Users string - Alias string - PubSubs string - LoadHistory string - Cdrs string - SMCosts string - ResourceProfiles string - Timings string -} - -type MigrationInfo struct { - Prefix string - DbVersion string - CurrentVersion string -} - -func (sv *StructVersion) CompareAndMigrate(dbVer *StructVersion) []*MigrationInfo { - var migrationInfoList []*MigrationInfo - if sv.Destinations != dbVer.Destinations { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.DESTINATION_PREFIX, - DbVersion: dbVer.Destinations, - CurrentVersion: CurrentVersion.Destinations, - }) - - } - if sv.RatingPlans != dbVer.RatingPlans { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.RATING_PLAN_PREFIX, - DbVersion: dbVer.RatingPlans, - CurrentVersion: CurrentVersion.RatingPlans, - }) - } - if sv.RatingProfiles != dbVer.RatingProfiles { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.RATING_PROFILE_PREFIX, - DbVersion: dbVer.RatingProfiles, - CurrentVersion: CurrentVersion.RatingProfiles, - }) - } - if sv.Lcrs != dbVer.Lcrs { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.LCR_PREFIX, - DbVersion: dbVer.Lcrs, - CurrentVersion: CurrentVersion.Lcrs, - }) - } - if sv.DerivedChargers != dbVer.DerivedChargers { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.DERIVEDCHARGERS_PREFIX, - DbVersion: dbVer.DerivedChargers, - CurrentVersion: CurrentVersion.DerivedChargers, - }) - } - if sv.Actions != dbVer.Actions { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.ACTION_PREFIX, - DbVersion: dbVer.Actions, - CurrentVersion: CurrentVersion.Actions, - }) - } - if sv.ActionPlans != dbVer.ActionPlans { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.ACTION_PLAN_PREFIX, - DbVersion: dbVer.ActionPlans, - CurrentVersion: CurrentVersion.ActionPlans, - }) - } - if sv.ActionTriggers != dbVer.ActionTriggers { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.ACTION_TRIGGER_PREFIX, - DbVersion: dbVer.ActionTriggers, - CurrentVersion: CurrentVersion.ActionTriggers, - }) - } - if sv.SharedGroups != dbVer.SharedGroups { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.SHARED_GROUP_PREFIX, - DbVersion: dbVer.SharedGroups, - CurrentVersion: CurrentVersion.SharedGroups, - }) - } - if sv.Accounts != dbVer.Accounts { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.ACCOUNT_PREFIX, - DbVersion: dbVer.Accounts, - CurrentVersion: CurrentVersion.Accounts, - }) - } - if sv.CdrStats != dbVer.CdrStats { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.CDR_STATS_PREFIX, - DbVersion: dbVer.CdrStats, - CurrentVersion: CurrentVersion.CdrStats, - }) - } - if sv.Users != dbVer.Users { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.USERS_PREFIX, - DbVersion: dbVer.Users, - CurrentVersion: CurrentVersion.Users, - }) - } - if sv.Alias != dbVer.Alias { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.ALIASES_PREFIX, - DbVersion: dbVer.Alias, - CurrentVersion: CurrentVersion.Alias, - }) - } - if sv.PubSubs != dbVer.PubSubs { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.PUBSUB_SUBSCRIBERS_PREFIX, - DbVersion: dbVer.PubSubs, - CurrentVersion: CurrentVersion.PubSubs, - }) - } - if sv.LoadHistory != dbVer.LoadHistory { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.LOADINST_KEY, - DbVersion: dbVer.LoadHistory, - CurrentVersion: CurrentVersion.LoadHistory, - }) - } - if sv.Cdrs != dbVer.Cdrs { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.CDRS_SOURCE, - DbVersion: dbVer.RatingPlans, - CurrentVersion: CurrentVersion.RatingPlans, - }) - } - if sv.SMCosts != dbVer.SMCosts { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.SMG, - DbVersion: dbVer.SMCosts, - CurrentVersion: CurrentVersion.SMCosts, - }) - } - if sv.ResourceProfiles != dbVer.ResourceProfiles { - migrationInfoList = append(migrationInfoList, &MigrationInfo{ - Prefix: utils.ResourceProfilesPrefix, - DbVersion: dbVer.ResourceProfiles, - CurrentVersion: CurrentVersion.ResourceProfiles, - }) - } - return migrationInfoList ->>>>>>> 25babe9746da2624d951a4f7d7b7babdfd646479 } func CurrentDataDBVersions() Versions { diff --git a/engine/version_test.go b/engine/version_test.go index 8d9c52828..e2644d352 100644 --- a/engine/version_test.go +++ b/engine/version_test.go @@ -30,19 +30,19 @@ func TestVersionCompare(t *testing.T) { q := Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 1, utils.COST_DETAILS: 2} c := Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 1} - message1 := y.Compare(x) + message1 := y.Compare(x, utils.MONGO) if message1 != "cgr-migrator -migrate=*accounts" { t.Errorf("Error failed to compare to curent version expected: %s received: %s", "cgr-migrator -migrate=*accounts", message1) } - message2 := z.Compare(x) + message2 := z.Compare(x, utils.MONGO) if message2 != "cgr-migrator -migrate=*action_plans" { t.Errorf("Error failed to compare to curent version expected: %s received: %s", "cgr-migrator -migrate=*action_plans", message2) } - message3 := q.Compare(x) + message3 := q.Compare(x, utils.MONGO) if message3 != "cgr-migrator -migrate=*shared_groups" { t.Errorf("Error failed to compare to curent version expected: %s received: %s", "cgr-migrator -migrate=*shared_groups", message3) } - message4 := c.Compare(x) + message4 := c.Compare(x, utils.MONGO) if message4 != "cgr-migrator -migrate=*cost_details" { t.Errorf("Error failed to compare to curent version expected: %s received: %s", "cgr-migrator -migrate=*cost_details", message4) } diff --git a/engine/versions_it_test.go b/engine/versions_it_test.go index 49abb240a..603e57280 100644 --- a/engine/versions_it_test.go +++ b/engine/versions_it_test.go @@ -142,9 +142,32 @@ func testVersionsFlush(t *testing.T) { } func TestVersion(t *testing.T) { - test := "Migration needed: please backup cgr data and run : " - currentVersion := Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} - testVersion := Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} + var test string + var currentVersion Versions + var testVersion Versions + storType := dataDb.GetStorageType() + switch storType { + case utils.MONGO: + currentVersion = Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} + testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} + test = "Migration needed: please backup cgr data and run : " + case utils.POSTGRES: + currentVersion = CurrentStorDBVersions() + testVersion = Versions{utils.COST_DETAILS: 1} + test = "Migration needed: please backup cgr data and run : " + case utils.MYSQL: + currentVersion = CurrentStorDBVersions() + testVersion = Versions{utils.COST_DETAILS: 1} + test = "Migration needed: please backup cgr data and run : " + case utils.REDIS: + currentVersion = CurrentDataDBVersions() + testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2} + test = "Migration needed: please backup cgr data and run : " + case utils.MAPSTOR: + currentVersion = Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} + testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} + test = "Migration needed: please backup cgr data and run : " + } //dataDB if _, rcvErr := dataDb.GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound { @@ -174,6 +197,29 @@ func TestVersion(t *testing.T) { t.Error(err) } + storType = storDb.GetStorageType() + switch storType { + case utils.MONGO: + currentVersion = Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} + testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} + test = "Migration needed: please backup cgr data and run : " + case utils.POSTGRES: + currentVersion = CurrentStorDBVersions() + testVersion = Versions{utils.COST_DETAILS: 1} + test = "Migration needed: please backup cgr data and run : " + case utils.MYSQL: + currentVersion = CurrentStorDBVersions() + testVersion = Versions{utils.COST_DETAILS: 1} + test = "Migration needed: please backup cgr data and run : " + case utils.REDIS: + currentVersion = CurrentDataDBVersions() + testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2} + test = "Migration needed: please backup cgr data and run : " + case utils.MAPSTOR: + currentVersion = Versions{utils.Accounts: 2, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} + testVersion = Versions{utils.Accounts: 1, utils.Actions: 2, utils.ActionTriggers: 2, utils.ActionPlans: 2, utils.SharedGroups: 2, utils.COST_DETAILS: 2} + test = "Migration needed: please backup cgr data and run : " + } //storDB if _, rcvErr := storDb.GetVersions(utils.TBLVersions); rcvErr != utils.ErrNotFound { t.Error(rcvErr) diff --git a/migrator/migrator.go b/migrator/migrator.go index 1e8222f5f..25f1baf98 100755 --- a/migrator/migrator.go +++ b/migrator/migrator.go @@ -23,7 +23,7 @@ import ( "github.com/cgrates/cgrates/utils" ) -func NewMigrator(dataDB engine.DataDB, dataDBType, dataDBEncoding string, storDB engine.Storage, storDBType string, oldDataDB v1DataDB, oldDataDBType, oldDataDBEncoding string, oldStorDB engine.Storage, oldStorDBType string) (m *Migrator, err error) { +func NewMigrator(dataDB engine.DataDB, dataDBType, dataDBEncoding string, storDB engine.Storage, storDBType string, oldDataDB V1DataDB, oldDataDBType, oldDataDBEncoding string, oldStorDB engine.Storage, oldStorDBType string) (m *Migrator, err error) { var mrshlr engine.Marshaler var oldmrshlr engine.Marshaler if dataDBEncoding == utils.MSGPACK { @@ -50,7 +50,7 @@ type Migrator struct { storDB engine.Storage storDBType string mrshlr engine.Marshaler - oldDataDB v1DataDB + oldDataDB V1DataDB oldDataDBType string oldStorDB engine.Storage oldStorDBType string @@ -66,12 +66,19 @@ func (m *Migrator) Migrate(taskID string) (err error) { utils.UnsupportedMigrationTask, fmt.Sprintf("task <%s> is not a supported migration task", taskID)) case utils.MetaSetVersions: - if err := m.storDB.SetVersions(engine.CurrentStorDBVersions(), false); err != nil { + if err := m.storDB.SetVersions(engine.CurrentDBVersions(m.storDBType), true); err != nil { return utils.NewCGRError(utils.Migrator, utils.ServerErrorCaps, err.Error(), fmt.Sprintf("error: <%s> when updating CostDetails version into StorDB", err.Error())) } + if err := m.dataDB.SetVersions(engine.CurrentDBVersions(m.dataDBType), true); err != nil { + return utils.NewCGRError(utils.Migrator, + utils.ServerErrorCaps, + err.Error(), + fmt.Sprintf("error: <%s> when updating CostDetails version into StorDB", err.Error())) + } + case utils.MetaCostDetails: err = m.migrateCostDetails() case utils.MetaAccounts: diff --git a/migrator/v1DataDB.go b/migrator/v1DataDB.go index 27b4c8f6e..6f16e525a 100644 --- a/migrator/v1DataDB.go +++ b/migrator/v1DataDB.go @@ -17,7 +17,7 @@ along with this program. If not, see */ package migrator -type v1DataDB interface { +type V1DataDB interface { getKeysForPrefix(prefix string) ([]string, error) getv1Account() (v1Acnt *v1Account, err error) setV1Account(x *v1Account) (err error) diff --git a/migrator/v1Migrator_Utils.go b/migrator/v1Migrator_Utils.go index 88a786898..f46bc71a0 100644 --- a/migrator/v1Migrator_Utils.go +++ b/migrator/v1Migrator_Utils.go @@ -25,8 +25,8 @@ import ( "github.com/cgrates/cgrates/utils" ) -func ConfigureV1DataStorage(db_type, host, port, name, user, pass, marshaler string) (db v1DataDB, err error) { - var d v1DataDB +func ConfigureV1DataStorage(db_type, host, port, name, user, pass, marshaler string) (db V1DataDB, err error) { + var d V1DataDB switch db_type { case utils.REDIS: var db_nb int @@ -40,8 +40,8 @@ func ConfigureV1DataStorage(db_type, host, port, name, user, pass, marshaler str } d, err = newv1RedisStorage(host, db_nb, pass, marshaler) case utils.MONGO: - d, err = NewMongoStorage(host, port, name, user, pass, utils.DataDB, nil) - db = d.(v1DataDB) + d, err = newv1MongoStorage(host, port, name, user, pass, utils.DataDB, nil) + db = d.(V1DataDB) default: err = errors.New(fmt.Sprintf("Unknown db '%s' valid options are '%s' or '%s'", db_type, utils.REDIS, utils.MONGO)) @@ -51,3 +51,23 @@ func ConfigureV1DataStorage(db_type, host, port, name, user, pass, marshaler str } return d, nil } + +// func ConfigureV1Storage(db_type, host, port, name, user, pass, marshaler string) (db v1StorDB, err error) { +// func ConfigureStorStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn, connMaxLifetime int, cdrsIndexes []string) (db Storage, err error) { +// var d Storage +// switch db_type { +// case utils.MONGO: +// d, err = newv1MongoStorage(host, port, name, user, pass, utils.StorDB, cdrsIndexes, nil, 1) +// case utils.POSTGRES: +// d, err = NewPostgresStorage(host, port, name, user, pass, maxConn, maxIdleConn, connMaxLifetime) +// case utils.MYSQL: +// d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn, connMaxLifetime) +// default: +// err = errors.New(fmt.Sprintf("Unknown db '%s' valid options are [%s, %s, %s]", +// db_type, utils.MYSQL, utils.MONGO, utils.POSTGRES)) +// } +// if err != nil { +// return nil, err +// } +// return d, nil +// } diff --git a/migrator/v1MongoData.go b/migrator/v1MongoData.go index f332cb956..9221beb0a 100644 --- a/migrator/v1MongoData.go +++ b/migrator/v1MongoData.go @@ -19,11 +19,9 @@ package migrator import ( "fmt" - // "log" - - "github.com/cgrates/cgrates/utils" "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" "gopkg.in/mgo.v2" ) @@ -43,7 +41,7 @@ type AtKeyValue struct { Value v1ActionPlans } -func NewMongoStorage(host, port, db, user, pass, storageType string, cdrsIndexes []string) (v1ms *v1Mongo, err error) { +func newv1MongoStorage(host, port, db, user, pass, storageType string, cdrsIndexes []string) (v1ms *v1Mongo, err error) { url := host if port != "" { url += ":" + port diff --git a/migrator/v1MongoStor.go b/migrator/v1MongoStor.go index 12fa903fd..eadea7570 100644 --- a/migrator/v1MongoStor.go +++ b/migrator/v1MongoStor.go @@ -16,3 +16,10 @@ You should have received a copy of the GNU General Public License along with this program. If not, see */ package migrator + +// import ( +// "github.com/cgrates/cgrates/utils" +// ) +// getV1CallCost() (v1CC *v1CallCost, err error){ +// return utils.ErrNotImplemented +// } diff --git a/migrator/v1Redis.go b/migrator/v1Redis.go index 19eb7deeb..ff107bd40 100644 --- a/migrator/v1Redis.go +++ b/migrator/v1Redis.go @@ -19,7 +19,6 @@ package migrator import ( "fmt" - //"log" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" diff --git a/migrator/v1SqlStor.go b/migrator/v1SqlStor.go index 12fa903fd..6642cf1a2 100644 --- a/migrator/v1SqlStor.go +++ b/migrator/v1SqlStor.go @@ -16,3 +16,108 @@ You should have received a copy of the GNU General Public License along with this program. If not, see */ package migrator + +// import ( +// "fmt" +// "time" + +// _ "github.com/go-sql-driver/mysql" +// "github.com/cgrates/cgrates/utils" +// "github.com/jinzhu/gorm" +// _ "github.com/lib/pq" +// ) + +// type v1SQLStorage struct { +// Db *sql.DB +// db *gorm.DB +// Storsql +// } + +// func NewPostgresStorage(host, port, name, user, password string, maxConn, maxIdleConn, connMaxLifetime int) (*v1SQLStorage, error) { +// connectString := fmt.Sprintf("host=%s port=%s dbname=%s user=%s password=%s sslmode=disable", host, port, name, user, password) +// db, err := gorm.Open("postgres", connectString) +// if err != nil { +// return nil, err +// } +// err = db.DB().Ping() +// if err != nil { +// return nil, err +// } +// db.DB().SetMaxIdleConns(maxIdleConn) +// db.DB().SetMaxOpenConns(maxConn) +// db.DB().SetConnMaxLifetime(time.Duration(connMaxLifetime) * time.Second) +// //db.LogMode(true) +// postgressStorage := new(PostgresStorage) +// postgressStorage.db = db +// postgressStorage.Db = db.DB() +// return &SQLStorage{db.DB(), db, postgressStorage, postgressStorage}, nil +// } + +// func NewMySQLStorage(host, port, name, user, password string, maxConn, maxIdleConn, connMaxLifetime int) (*v1SQLStorage, error) { +// connectString := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&loc=Local&parseTime=true", user, password, host, port, name) +// db, err := gorm.Open("mysql", connectString) +// if err != nil { +// return nil, err +// } +// if err = db.DB().Ping(); err != nil { +// return nil, err +// } +// db.DB().SetMaxIdleConns(maxIdleConn) +// db.DB().SetMaxOpenConns(maxConn) +// db.DB().SetConnMaxLifetime(time.Duration(connMaxLifetime) * time.Second) +// //db.LogMode(true) +// mySQLStorage := new(MySQLStorage) +// mySQLStorage.db = db +// mySQLStorage.Db = db.DB() +// return &SQLStorage{db.DB(), db, mySQLStorage, mySQLStorage}, nil +// } + +// getV1CallCost() (v1CC *v1CallCost, err error){ +// //echivalentu la ce am facut la mongo doar ca cu rows + +// var storSQL *sql.DB +// switch m.storDBType { +// case utils.MYSQL: +// storSQL = m.storDB.(*engine.SQLStorage).Db +// case utils.POSTGRES: +// storSQL = m.storDB.(*engine.PostgresStorage).Db +// default: +// return utils.NewCGRError(utils.Migrator, +// utils.MandatoryIEMissingCaps, +// utils.UnsupportedDB, +// fmt.Sprintf("unsupported database type: <%s>", m.storDBType)) +// } +// rows, err := storSQL.Query("SELECT id, tor, direction, tenant, category, account, subject, destination, cost, cost_details FROM cdrs") +// if err != nil { +// return utils.NewCGRError(utils.Migrator, +// utils.ServerErrorCaps, +// err.Error(), +// fmt.Sprintf("error: <%s> when querying storDB for cdrs", err.Error())) +// } + +// defer rows.Close() + +// for cnt := 0; rows.Next(); cnt++ { +// var id int64 +// var ccDirection, ccCategory, ccTenant, ccSubject, ccAccount, ccDestination, ccTor sql.NullString +// var ccCost sql.NullFloat64 +// var tts []byte + +// if err := rows.Scan(&id, &ccTor, &ccDirection, &ccTenant, &ccCategory, &ccAccount, &ccSubject, &ccDestination, &ccCost, &tts); err != nil { +// return utils.NewCGRError(utils.Migrator, +// utils.ServerErrorCaps, +// err.Error(), +// fmt.Sprintf("error: <%s> when scanning at count: <%d>", err.Error(), cnt)) +// } +// var v1tmsps v1TimeSpans +// if err := json.Unmarshal(tts, &v1tmsps); err != nil { +// utils.Logger.Warning( +// fmt.Sprintf(" Unmarshalling timespans at CDR with id: <%d>, error: <%s>", id, err.Error())) +// continue +// } +// return v1CC := &v1CallCost{Direction: ccDirection.String, Category: ccCategory.String, Tenant: ccTenant.String, +// Subject: ccSubject.String, Account: ccAccount.String, Destination: ccDestination.String, TOR: ccTor.String, +// Cost: ccCost.Float64, Timespans: v1tmsps} + +// } +// } diff --git a/migrator/v1StorDB.go b/migrator/v1StorDB.go index 20a074281..c83369852 100644 --- a/migrator/v1StorDB.go +++ b/migrator/v1StorDB.go @@ -18,5 +18,5 @@ along with this program. If not, see package migrator // type v1StorDB interface { - +// getV1CallCost() (v1CC *v1CallCost, err error) // } diff --git a/utils/consts.go b/utils/consts.go index c627438ec..94617b365 100755 --- a/utils/consts.go +++ b/utils/consts.go @@ -73,6 +73,7 @@ const ( MYSQL = "mysql" MONGO = "mongo" REDIS = "redis" + MAPSTOR = "mapstor" LOCALHOST = "127.0.0.1" FSCDR_FILE_CSV = "freeswitch_file_csv" FSCDR_HTTP_JSON = "freeswitch_http_json" @@ -409,10 +410,10 @@ const ( MetaFileCSV = "*file_csv" MetaFileFWV = "*file_fwv" Accounts = "Accounts" - Actions = "Actions" - ActionPlans = "ActionPlans" - ActionTriggers = "ActionTriggers" - SharedGroups = "SharedGroups" + Actions = "Actions" + ActionPlans = "ActionPlans" + ActionTriggers = "ActionTriggers" + SharedGroups = "SharedGroups" MetaEveryMinute = "*every_minute" MetaHourly = "*hourly" ID = "ID" diff --git a/utils/errors.go b/utils/errors.go index 5e079f55a..42cf41039 100644 --- a/utils/errors.go +++ b/utils/errors.go @@ -23,7 +23,7 @@ import ( ) var ( - ErrNoMoreData = errors.New("NO_MORE_DATA") + ErrNoMoreData = errors.New("NO_MORE_DATA") ErrNotImplemented = errors.New("NOT_IMPLEMENTED") ErrNotFound = errors.New("NOT_FOUND") ErrTimedOut = errors.New("TIMED_OUT") diff --git a/utils/map.go b/utils/map.go index 1c3d11c46..0c30e4868 100644 --- a/utils/map.go +++ b/utils/map.go @@ -18,8 +18,8 @@ along with this program. If not, see package utils import ( - "strings" - "strconv" + "strconv" + "strings" ) // Converts map[string]string into map[string]interface{} @@ -220,14 +220,14 @@ func (fmp FieldMultiplyFactor) Clone() (cln FieldMultiplyFactor) { return } -func MapStringToInt64(in map[string]string) (out map[string]int64, err error){ -mapout := make(map[string]int64, len(in)) +func MapStringToInt64(in map[string]string) (out map[string]int64, err error) { + mapout := make(map[string]int64, len(in)) for key, val := range in { - x,err:=strconv.Atoi(val) - if err!=nil{ - return nil,err + x, err := strconv.Atoi(val) + if err != nil { + return nil, err } mapout[key] = int64(x) } - return mapout,nil + return mapout, nil } diff --git a/utils/map_test.go b/utils/map_test.go index d58f3639f..ccd576451 100644 --- a/utils/map_test.go +++ b/utils/map_test.go @@ -92,14 +92,14 @@ func TestIsEmpty(t *testing.T) { } } -func TestMapStringToInt64(t *testing.T){ - t1:=map[string]int64{"test":int64(21)} - t2:=map[string]string{"test":"21"} - t3,err:=MapStringToInt64(t2) - if err!=nil{ - t.Error("Got Error: ",err) +func TestMapStringToInt64(t *testing.T) { + t1 := map[string]int64{"test": int64(21)} + t2 := map[string]string{"test": "21"} + t3, err := MapStringToInt64(t2) + if err != nil { + t.Error("Got Error: ", err) } - if !reflect.DeepEqual(t1,t3){ + if !reflect.DeepEqual(t1, t3) { t.Errorf("Expecting: %+v, received: %+v", t1, t3) } -} \ No newline at end of file +}