From aa3a39b78383c1d45c040e73a854f27d2be70228 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Fri, 9 Oct 2015 18:55:12 +0300 Subject: [PATCH 1/7] most of mongo tp work in progress --- engine/storage_mongo_tp.go | 603 +++++++++++++++++++++++++++++++++++++ engine/storage_sql.go | 9 +- 2 files changed, 611 insertions(+), 1 deletion(-) create mode 100644 engine/storage_mongo_tp.go diff --git a/engine/storage_mongo_tp.go b/engine/storage_mongo_tp.go new file mode 100644 index 000000000..ca2c90eba --- /dev/null +++ b/engine/storage_mongo_tp.go @@ -0,0 +1,603 @@ +package engine + +import ( + "github.com/cgrates/cgrates/utils" + "gopkg.in/mgo.v2/bson" +) + +const ( + colTpTmg = "tp_timings" + colTpDst = "tp_destinations" + colTpRts = "tp_rates" + colTpDrs = "tp_destinationrates" + colTpRpl = "tp_ratingplans" + colTpRpf = "tp_ratingprofiles" + colTpAct = "tp_actions" + colTpApl = "tp_actionplans" + colTpAtr = "tp_actiontriggers" + colTpAcc = "tp_accounts" + colTpShg = "tp_sharedgroups" + colTpLcr = "tp_lcrrules" + colTpDcs = "tp_derivedchargers" + colTpAls = "tp_aliases" + colTpStq = "tp_statsqeues" + colTpPbs = "tp_pubsub" + colTpUsr = "tp_users" + colTpCrs = "tp_cdrstats" +) + +func (ms *MongoStorage) GetTpIds() ([]string, error) { return nil, nil } +func (ms *MongoStorage) GetTpTableIds(string, string, utils.TPDistinctIds, map[string]string, *utils.Paginator) ([]string, error) { + return nil, nil +} +func (ms *MongoStorage) GetTpTimings(tpid, tag string) ([]TpTiming, error) { + filter := bson.M{ + "tpid": tpid, + } + if tag != "" { + filter["tag"] = tag + } + var results []TpTiming + err := ms.db.C(colTpTmg).Find(filter).All(&results) + return results, err +} +func (ms *MongoStorage) GetTpDestinations(tpid, tag string) ([]TpDestination, error) { + filter := bson.M{ + "tpid": tpid, + } + if tag != "" { + filter["tag"] = tag + } + var results []TpDestination + err := ms.db.C(colTpDst).Find(filter).All(&results) + return results, err +} +func (ms *MongoStorage) GetTpRates(tpid, tag string) ([]TpRate, error) { + filter := bson.M{ + "tpid": tpid, + } + if tag != "" { + filter["tag"] = tag + } + var results []TpRate + err := ms.db.C(colTpRts).Find(filter).All(&results) + return results, err +} +func (ms *MongoStorage) GetTpDestinationRates(tpid, tag string, pag *utils.Paginator) ([]TpDestinationRate, error) { + filter := bson.M{ + "tpid": tpid, + } + if tag != "" { + filter["tag"] = tag + } + var results []TpDestinationRate + q := ms.db.C(colTpDrs).Find(filter) + if pag != nil { + if pag.Limit != nil { + q = q.Limit(*pag.Limit) + } + if pag.Offset != nil { + q = q.Skip(*pag.Offset) + } + } + err := q.All(&results) + return results, err +} + +func (ms *MongoStorage) GetTpRatingPlans(tpid, tag string, pag *utils.Paginator) ([]TpRatingPlan, error) { + filter := bson.M{ + "tpid": tpid, + } + if tag != "" { + filter["tag"] = tag + } + var results []TpRatingPlan + q := ms.db.C(colTpRpl).Find(filter) + if pag != nil { + if pag.Limit != nil { + q = q.Limit(*pag.Limit) + } + if pag.Offset != nil { + q = q.Skip(*pag.Offset) + } + } + err := q.All(&results) + return results, err +} + +func (ms *MongoStorage) GetTpRatingProfiles(tp *TpRatingProfile) ([]TpRatingProfile, error) { + filter := bson.M{"tpid": tp.Tpid} + if tp.Direction != "" { + filter["direction"] = tp.Direction + } + if tp.Tenant != "" { + filter["tenant"] = tp.Tenant + } + if tp.Category != "" { + filter["category"] = tp.Category + } + if tp.Subject != "" { + filter["subject"] = tp.Subject + } + if tp.Loadid != "" { + filter["loadid"] = tp.Loadid + } + var results []TpRatingProfile + err := ms.db.C(colTpRpf).Find(filter).All(&results) + return results, err +} + +func (ms *MongoStorage) GetTpSharedGroups(tpid, tag string) ([]TpSharedGroup, error) { + filter := bson.M{ + "tpid": tpid, + } + if tag != "" { + filter["tag"] = tag + } + var results []TpSharedGroup + err := ms.db.C(colTpShg).Find(filter).All(&results) + return results, err +} + +func (ms *MongoStorage) GetTpCdrStats(tpid, tag string) ([]TpCdrstat, error) { + filter := bson.M{ + "tpid": tpid, + } + if tag != "" { + filter["tag"] = tag + } + var results []TpCdrstat + err := ms.db.C(colTpCrs).Find(filter).All(&results) + return results, err +} +func (ms *MongoStorage) GetTpLCRs(tp *TpLcrRule) ([]TpLcrRule, error) { + filter := bson.M{"tpid": tp.Tpid} + if tp.Direction != "" { + filter["direction"] = tp.Direction + } + if tp.Tenant != "" { + filter["tenant"] = tp.Tenant + } + if tp.Category != "" { + filter["category"] = tp.Category + } + if tp.Account != "" { + filter["account"] = tp.Account + } + if tp.Subject != "" { + filter["subject"] = tp.Subject + } + var results []TpLcrRule + err := ms.db.C(colTpLcr).Find(filter).All(&results) + return results, err +} + +func (ms *MongoStorage) GetTpUsers(tp *TpUser) ([]TpUser, error) { + filter := bson.M{"tpid": tp.Tpid} + if tp.Tenant != "" { + filter["tenant"] = tp.Tenant + } + if tp.UserName != "" { + filter["username"] = tp.UserName + } + var results []TpUser + err := ms.db.C(colTpUsr).Find(filter).All(&results) + return results, err +} + +func (ms *MongoStorage) GetTpAliases(tp *TpAlias) ([]TpAlias, error) { + filter := bson.M{"tpid": tp.Tpid} + if tp.Direction != "" { + filter["direction"] = tp.Direction + } + if tp.Tenant != "" { + filter["tenant"] = tp.Tenant + } + if tp.Category != "" { + filter["category"] = tp.Category + } + if tp.Account != "" { + filter["account"] = tp.Account + } + if tp.Subject != "" { + filter["subject"] = tp.Subject + } + if tp.Context != "" { + filter["context"] = tp.Context + } + var results []TpAlias + err := ms.db.C(colTpAls).Find(filter).All(&results) + return results, err +} + +func (ms *MongoStorage) GetTpDerivedChargers(tp *TpDerivedCharger) ([]TpDerivedCharger, error) { + filter := bson.M{"tpid": tp.Tpid} + if tp.Direction != "" { + filter["direction"] = tp.Direction + } + if tp.Tenant != "" { + filter["tenant"] = tp.Tenant + } + if tp.Category != "" { + filter["category"] = tp.Category + } + if tp.Subject != "" { + filter["subject"] = tp.Subject + } + if tp.Account != "" { + filter["account"] = tp.Account + } + if tp.Loadid != "" { + filter["loadid"] = tp.Loadid + } + var results []TpDerivedCharger + err := ms.db.C(colTpDcs).Find(filter).All(&results) + return results, err +} + +func (ms *MongoStorage) GetTpActions(tpid, tag string) ([]TpAction, error) { + filter := bson.M{ + "tpid": tpid, + } + if tag != "" { + filter["tag"] = tag + } + var results []TpAction + err := ms.db.C(colTpAct).Find(filter).All(&results) + return results, err +} + +func (ms *MongoStorage) GetTpActionPlans(tpid, tag string) ([]TpActionPlan, error) { + filter := bson.M{ + "tpid": tpid, + } + if tag != "" { + filter["tag"] = tag + } + var results []TpActionPlan + err := ms.db.C(colTpApl).Find(filter).All(&results) + return results, err +} + +func (ms *MongoStorage) GetTpActionTriggers(tpid, tag string) ([]TpActionTrigger, error) { + filter := bson.M{ + "tpid": tpid, + } + if tag != "" { + filter["tag"] = tag + } + var results []TpActionTrigger + err := ms.db.C(colTpAtr).Find(filter).All(&results) + return results, err +} + +func (ms *MongoStorage) GetTpAccountActions(tp *TpAccountAction) ([]TpAccountAction, error) { + filter := bson.M{"tpid": tp.Tpid} + if tp.Direction != "" { + filter["direction"] = tp.Direction + } + if tp.Tenant != "" { + filter["tenant"] = tp.Tenant + } + if tp.Account != "" { + filter["account"] = tp.Account + } + if tp.Loadid != "" { + filter["loadid"] = tp.Loadid + } + var results []TpAccountAction + err := ms.db.C(colTpAcc).Find(filter).All(&results) + return results, err +} + +func (ms *MongoStorage) RemTpData(string, string, ...string) error { + + return nil +} + +func (ms *MongoStorage) SetTpTimings(tps []TpTiming) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpTmg).Bulk() + for _, tp := range tps { + if found, _ := m[tp.Tag]; !found { + m[tp.Tag] = true + tx.Upsert(bson.M{"tpid": tp.Tpid, "tag": tp.Tag}, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpDestinations(tps []TpDestination) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpDst).Bulk() + for _, tp := range tps { + if found, _ := m[tp.Tag]; !found { + m[tp.Tag] = true + tx.Upsert(bson.M{"tpid": tp.Tpid, "tag": tp.Tag}, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpRates(tps []TpRate) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpRts).Bulk() + for _, tp := range tps { + if found, _ := m[tp.Tag]; !found { + m[tp.Tag] = true + tx.Upsert(bson.M{"tpid": tp.Tpid, "tag": tp.Tag}, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpDestinationRates(tps []TpDestinationRate) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpDrs).Bulk() + for _, tp := range tps { + if found, _ := m[tp.Tag]; !found { + m[tp.Tag] = true + tx.Upsert(bson.M{"tpid": tp.Tpid, "tag": tp.Tag}, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpRatingPlans(tps []TpRatingPlan) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpRpl).Bulk() + for _, tp := range tps { + if found, _ := m[tp.Tag]; !found { + m[tp.Tag] = true + tx.Upsert(bson.M{"tpid": tp.Tpid, "tag": tp.Tag}, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpRatingProfiles(tps []TpRatingProfile) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpRpf).Bulk() + for _, tp := range tps { + if found, _ := m[tp.GetRatingProfileId()]; !found { + m[tp.GetRatingProfileId()] = true + tx.Upsert(bson.M{ + "tpid": tp.Tpid, + "loadid": tp.Loadid, + "direction": tp.Direction, + "tenant": tp.Tenant, + "category": tp.Category, + "subject": tp.Subject, + }, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpSharedGroups(tps []TpSharedGroup) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpShg).Bulk() + for _, tp := range tps { + if found, _ := m[tp.Tag]; !found { + m[tp.Tag] = true + tx.Upsert(bson.M{"tpid": tp.Tpid, "tag": tp.Tag}, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpCdrStats(tps []TpCdrstat) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpCrs).Bulk() + for _, tp := range tps { + if found, _ := m[tp.Tag]; !found { + m[tp.Tag] = true + tx.Upsert(bson.M{"tpid": tp.Tpid, "tag": tp.Tag}, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpUsers(tps []TpUser) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpUsr).Bulk() + for _, tp := range tps { + if found, _ := m[tp.GetId()]; !found { + m[tp.GetId()] = true + tx.Upsert(bson.M{ + "tpid": tp.Tpid, + "tenant": tp.Tenant, + "username": tp.UserName, + }, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpAliases(tps []TpAlias) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpAls).Bulk() + for _, tp := range tps { + if found, _ := m[tp.GetId()]; !found { + m[tp.GetId()] = true + tx.Upsert(bson.M{ + "tpid": tp.Tpid, + "direction": tp.Direction, + "tenant": tp.Tenant, + "category": tp.Category, + "account": tp.Account, + "subject": tp.Subject, + "context": tp.Context}, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpDerivedChargers(tps []TpDerivedCharger) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpRpl).Bulk() + for _, tp := range tps { + if found, _ := m[tp.GetDerivedChargersId()]; !found { + m[tp.GetDerivedChargersId()] = true + tx.Upsert(bson.M{ + "tpid": tp.Tpid, + "direction": tp.Direction, + "tenant": tp.Tenant, + "category": tp.Category, + "account": tp.Account, + "subject": tp.Subject}, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpLCRs(tps []TpLcrRule) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpRpl).Bulk() + for _, tp := range tps { + if found, _ := m[tp.GetLcrRuleId()]; !found { + m[tp.GetLcrRuleId()] = true + tx.Upsert(bson.M{ + "tpid": tp.Tpid, + "direction": tp.Direction, + "tenant": tp.Tenant, + "category": tp.Category, + "account": tp.Account, + "subject": tp.Subject}, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpActions(tps []TpAction) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpAct).Bulk() + for _, tp := range tps { + if found, _ := m[tp.Tag]; !found { + m[tp.Tag] = true + tx.Upsert(bson.M{"tpid": tp.Tpid, "tag": tp.Tag}, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpActionPlans(tps []TpActionPlan) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpApl).Bulk() + for _, tp := range tps { + if found, _ := m[tp.Tag]; !found { + m[tp.Tag] = true + tx.Upsert(bson.M{"tpid": tp.Tpid, "tag": tp.Tag}, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpActionTriggers(tps []TpActionTrigger) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpAtr).Bulk() + for _, tp := range tps { + if found, _ := m[tp.Tag]; !found { + m[tp.Tag] = true + tx.Upsert(bson.M{"tpid": tp.Tpid, "tag": tp.Tag}, tp) + } + } + _, err := tx.Run() + return err +} + +func (ms *MongoStorage) SetTpAccountActions(tps []TpAccountAction) error { + if len(tps) == 0 { + return nil + } + m := make(map[string]bool) + + tx := ms.db.C(colTpAcc).Bulk() + for _, tp := range tps { + if found, _ := m[tp.GetAccountActionId()]; !found { + m[tp.GetAccountActionId()] = true + tx.Upsert(bson.M{ + "tpid": tp.Tpid, + "loadid": tp.Loadid, + "direction": tp.Direction, + "tenant": tp.Tenant, + "account": tp.Account}, tp) + } + } + _, err := tx.Run() + return err +} diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 0135c3892..179f8b799 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -410,7 +410,14 @@ func (self *SQLStorage) SetTpDerivedChargers(sgs []TpDerivedCharger) error { return err } - if err := tx.Where(tmpDc).Delete(TpDerivedCharger{}).Error; err != nil { + if err := tx.Where(tmpDc).Delete(TpDerivedCharger{ + Tpid: dCharger.Tpid, + Direction: dCharger.Direction, + Tenant: dCharger.Tenant, + Category: dCharger.Category, + Account: dCharger.Account, + Subject: dCharger.Subject, + }).Error; err != nil { tx.Rollback() return err } From 28561dba1f7eb1a294d6f2d9f6b176fb921da80b Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 12 Oct 2015 13:46:09 +0300 Subject: [PATCH 2/7] finshed mongo tp methods test pending --- apier/v1/tpaccountactions.go | 2 +- apier/v1/tpactionplans.go | 2 +- apier/v1/tpactions.go | 2 +- apier/v1/tpactiontriggers.go | 2 +- apier/v1/tpaliases.go | 2 +- apier/v1/tpcdrstats.go | 2 +- apier/v1/tpderivedcharges.go | 2 +- apier/v1/tpdestinationrates.go | 2 +- apier/v1/tpdestinations.go | 2 +- apier/v1/tplcrrules.go | 2 +- apier/v1/tprates.go | 2 +- apier/v1/tpratingplans.go | 2 +- apier/v1/tpratingprofiles.go | 2 +- apier/v1/tpsharedgroups.go | 2 +- apier/v1/tptimings.go | 2 +- apier/v1/tpusers.go | 2 +- apier/v2/tp.go | 2 +- engine/calldesc.go | 2 +- engine/storage_interface.go | 2 +- engine/storage_mongo_tp.go | 92 ++++++++++++++++++++++++++++-- engine/storage_mysql_local_test.go | 8 +-- engine/storage_psql_local_test.go | 8 +-- engine/storage_sql.go | 14 ++--- 23 files changed, 117 insertions(+), 43 deletions(-) diff --git a/apier/v1/tpaccountactions.go b/apier/v1/tpaccountactions.go index 56778e3d8..ab8bf2799 100644 --- a/apier/v1/tpaccountactions.go +++ b/apier/v1/tpaccountactions.go @@ -157,7 +157,7 @@ func (self *ApierV1) RemTPAccountActions(attrs AttrGetTPAccountActions, reply *s if err := aa.SetAccountActionId(attrs.AccountActionsId); err != nil { return err } - if err := self.StorDb.RemTpData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.Tpid, aa.Loadid, aa.Direction, aa.Tenant, aa.Account); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.Tpid, map[string]string{"loadid": aa.Loadid, "direction": aa.Direction, "tenant": aa.Tenant, "account": aa.Account}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tpactionplans.go b/apier/v1/tpactionplans.go index 4824c9794..9a09dd4fc 100644 --- a/apier/v1/tpactionplans.go +++ b/apier/v1/tpactionplans.go @@ -98,7 +98,7 @@ func (self *ApierV1) RemTPActionPlan(attrs AttrGetTPActionPlan, reply *string) e if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Id"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.StorDb.RemTpData(utils.TBL_TP_ACTION_PLANS, attrs.TPid, attrs.Id); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_ACTION_PLANS, attrs.TPid, map[string]string{"tag": attrs.Id}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tpactions.go b/apier/v1/tpactions.go index 1d53915fe..c09b7908b 100644 --- a/apier/v1/tpactions.go +++ b/apier/v1/tpactions.go @@ -96,7 +96,7 @@ func (self *ApierV1) RemTPActions(attrs AttrGetTPActions, reply *string) error { if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionsId"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.StorDb.RemTpData(utils.TBL_TP_ACTIONS, attrs.TPid, attrs.ActionsId); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_ACTIONS, attrs.TPid, map[string]string{"tag": attrs.ActionsId}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tpactiontriggers.go b/apier/v1/tpactiontriggers.go index 19447a849..7d153c5bb 100644 --- a/apier/v1/tpactiontriggers.go +++ b/apier/v1/tpactiontriggers.go @@ -92,7 +92,7 @@ func (self *ApierV1) RemTPActionTriggers(attrs AttrGetTPActionTriggers, reply *s if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionTriggersId"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.StorDb.RemTpData(utils.TBL_TP_ACTION_TRIGGERS, attrs.TPid, attrs.ActionTriggersId); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_ACTION_TRIGGERS, attrs.TPid, map[string]string{"tag": attrs.ActionTriggersId}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tpaliases.go b/apier/v1/tpaliases.go index cdb4b9296..1fc1b2276 100644 --- a/apier/v1/tpaliases.go +++ b/apier/v1/tpaliases.go @@ -87,7 +87,7 @@ func (self *ApierV1) RemTPAlias(attrs AttrGetTPAlias, reply *string) error { if missing := utils.MissingStructFields(&attrs, []string{"TPid", "AliasId"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.StorDb.RemTpData(utils.TBL_TP_ALIASES, attrs.TPid); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_ALIASES, attrs.TPid, map[string]string{"tag": attrs.AliasId}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tpcdrstats.go b/apier/v1/tpcdrstats.go index 9990cc139..7893602c9 100644 --- a/apier/v1/tpcdrstats.go +++ b/apier/v1/tpcdrstats.go @@ -94,7 +94,7 @@ func (self *ApierV1) RemTPCdrStats(attrs AttrGetTPCdrStats, reply *string) error if missing := utils.MissingStructFields(&attrs, []string{"TPid", "CdrStatsId"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.StorDb.RemTpData(utils.TBL_TP_SHARED_GROUPS, attrs.TPid, attrs.CdrStatsId); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_SHARED_GROUPS, attrs.TPid, map[string]string{"tag": attrs.CdrStatsId}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tpderivedcharges.go b/apier/v1/tpderivedcharges.go index 85b608581..e6d0c825b 100644 --- a/apier/v1/tpderivedcharges.go +++ b/apier/v1/tpderivedcharges.go @@ -103,7 +103,7 @@ func (self *ApierV1) RemTPDerivedChargers(attrs AttrGetTPDerivedChargers, reply if err := tmpDc.SetDerivedChargersId(attrs.DerivedChargersId); err != nil { return err } - if err := self.StorDb.RemTpData(utils.TBL_TP_DERIVED_CHARGERS, attrs.TPid, tmpDc.Loadid, tmpDc.Direction, tmpDc.Tenant, tmpDc.Category, tmpDc.Account, tmpDc.Subject); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_DERIVED_CHARGERS, attrs.TPid, map[string]string{"loadid": tmpDc.Loadid, "direction": tmpDc.Direction, "tenant": tmpDc.Tenant, "category": tmpDc.Category, "account": tmpDc.Account, "subject": tmpDc.Subject}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tpdestinationrates.go b/apier/v1/tpdestinationrates.go index 1fa003b40..e79af1504 100644 --- a/apier/v1/tpdestinationrates.go +++ b/apier/v1/tpdestinationrates.go @@ -88,7 +88,7 @@ func (self *ApierV1) RemTPDestinationRate(attrs AttrGetTPDestinationRate, reply if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationRateId"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.StorDb.RemTpData(utils.TBL_TP_DESTINATION_RATES, attrs.TPid, attrs.DestinationRateId); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_DESTINATION_RATES, attrs.TPid, map[string]string{"tag": attrs.DestinationRateId}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tpdestinations.go b/apier/v1/tpdestinations.go index a6f68fbeb..93aff6c3e 100644 --- a/apier/v1/tpdestinations.go +++ b/apier/v1/tpdestinations.go @@ -88,7 +88,7 @@ func (self *ApierV1) RemTPDestination(attrs AttrGetTPDestination, reply *string) if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationId"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.StorDb.RemTpData(utils.TBL_TP_DESTINATIONS, attrs.TPid, attrs.DestinationId); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_DESTINATIONS, attrs.TPid, map[string]string{"tag": attrs.DestinationId}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tplcrrules.go b/apier/v1/tplcrrules.go index 444339f2f..708e95329 100644 --- a/apier/v1/tplcrrules.go +++ b/apier/v1/tplcrrules.go @@ -89,7 +89,7 @@ func (self *ApierV1) RemTPLcrRule(attrs AttrGetTPLcrRules, reply *string) error if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LcrRulesId"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.StorDb.RemTpData(utils.TBL_TP_LCRS, attrs.TPid, attrs.LcrRuleId); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_LCRS, attrs.TPid, map[string]string{"tag": attrs.LcrRuleId}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tprates.go b/apier/v1/tprates.go index 64de7d1d4..5d87db447 100644 --- a/apier/v1/tprates.go +++ b/apier/v1/tprates.go @@ -87,7 +87,7 @@ func (self *ApierV1) RemTPRate(attrs AttrGetTPRate, reply *string) error { if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RateId"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.StorDb.RemTpData(utils.TBL_TP_RATES, attrs.TPid, attrs.RateId); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_RATES, attrs.TPid, map[string]string{"tag": attrs.RateId}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tpratingplans.go b/apier/v1/tpratingplans.go index 6bcdd0196..ff4e9831d 100644 --- a/apier/v1/tpratingplans.go +++ b/apier/v1/tpratingplans.go @@ -88,7 +88,7 @@ func (self *ApierV1) RemTPRatingPlan(attrs AttrGetTPRatingPlan, reply *string) e if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingPlanId"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.StorDb.RemTpData(utils.TBL_TP_RATING_PLANS, attrs.TPid, attrs.RatingPlanId); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_RATING_PLANS, attrs.TPid, map[string]string{"tag": attrs.RatingPlanId}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tpratingprofiles.go b/apier/v1/tpratingprofiles.go index c2cb75dba..7ea411f7d 100644 --- a/apier/v1/tpratingprofiles.go +++ b/apier/v1/tpratingprofiles.go @@ -161,7 +161,7 @@ func (self *ApierV1) RemTPRatingProfile(attrs AttrGetTPRatingProfile, reply *str if err := tmpRpf.SetRatingProfileId(attrs.RatingProfileId); err != nil { return err } - if err := self.StorDb.RemTpData(utils.TBL_TP_RATE_PROFILES, attrs.TPid, tmpRpf.Loadid, tmpRpf.Direction, tmpRpf.Tenant, tmpRpf.Category, tmpRpf.Subject); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_RATE_PROFILES, attrs.TPid, map[string]string{"loadid": tmpRpf.Loadid, "direction": tmpRpf.Direction, "tenant": tmpRpf.Tenant, "category": tmpRpf.Category, "subject": tmpRpf.Subject}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tpsharedgroups.go b/apier/v1/tpsharedgroups.go index 83880d614..881c7f1c9 100644 --- a/apier/v1/tpsharedgroups.go +++ b/apier/v1/tpsharedgroups.go @@ -94,7 +94,7 @@ func (self *ApierV1) RemTPSharedGroups(attrs AttrGetTPSharedGroups, reply *strin if missing := utils.MissingStructFields(&attrs, []string{"TPid", "SharedGroupsId"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.StorDb.RemTpData(utils.TBL_TP_SHARED_GROUPS, attrs.TPid, attrs.SharedGroupsId); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_SHARED_GROUPS, attrs.TPid, map[string]string{"tag": attrs.SharedGroupsId}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tptimings.go b/apier/v1/tptimings.go index dcd47421f..ad148674c 100644 --- a/apier/v1/tptimings.go +++ b/apier/v1/tptimings.go @@ -85,7 +85,7 @@ func (self *ApierV1) RemTPTiming(attrs AttrGetTPTiming, reply *string) error { if missing := utils.MissingStructFields(&attrs, []string{"TPid", "TimingId"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.StorDb.RemTpData(utils.TBL_TP_TIMINGS, attrs.TPid, attrs.TimingId); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_TIMINGS, attrs.TPid, map[string]string{"tag": attrs.TimingId}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v1/tpusers.go b/apier/v1/tpusers.go index 79d9ea2d2..44d07ae16 100644 --- a/apier/v1/tpusers.go +++ b/apier/v1/tpusers.go @@ -90,7 +90,7 @@ func (self *ApierV1) RemTPUser(attrs AttrGetTPUser, reply *string) error { if missing := utils.MissingStructFields(&attrs, []string{"TPid", "UserId"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := self.StorDb.RemTpData(utils.TBL_TP_USERS, attrs.TPid); err != nil { + if err := self.StorDb.RemTpData(utils.TBL_TP_USERS, attrs.TPid, map[string]string{"tag": attrs.UserId}); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/apier/v2/tp.go b/apier/v2/tp.go index 492eb85d5..75880df0e 100644 --- a/apier/v2/tp.go +++ b/apier/v2/tp.go @@ -33,7 +33,7 @@ func (self *ApierV2) RemTP(attrs AttrRemTp, reply *string) error { if len(attrs.TPid) == 0 { return utils.NewErrMandatoryIeMissing("TPid") } - if err := self.StorDb.RemTpData("", attrs.TPid); err != nil { + if err := self.StorDb.RemTpData("", attrs.TPid, nil); err != nil { return utils.NewErrServerError(err) } else { *reply = "OK" diff --git a/engine/calldesc.go b/engine/calldesc.go index 7fa0f5cce..eba37b91d 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -37,7 +37,7 @@ const ( RECURSION_MAX_DEPTH = 3 MIN_PREFIX_MATCH = 1 FALLBACK_SUBJECT = utils.ANY - DEBUG = true + DEBUG = false ) func init() { diff --git a/engine/storage_interface.go b/engine/storage_interface.go index ab5857c64..f18a652f3 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -137,7 +137,7 @@ type LoadReader interface { } type LoadWriter interface { - RemTpData(string, string, ...string) error + RemTpData(string, string, map[string]string) error SetTpTimings([]TpTiming) error SetTpDestinations([]TpDestination) error SetTpRates([]TpRate) error diff --git a/engine/storage_mongo_tp.go b/engine/storage_mongo_tp.go index ca2c90eba..4f345d5f1 100644 --- a/engine/storage_mongo_tp.go +++ b/engine/storage_mongo_tp.go @@ -1,6 +1,8 @@ package engine import ( + "strings" + "github.com/cgrates/cgrates/utils" "gopkg.in/mgo.v2/bson" ) @@ -26,10 +28,71 @@ const ( colTpCrs = "tp_cdrstats" ) -func (ms *MongoStorage) GetTpIds() ([]string, error) { return nil, nil } -func (ms *MongoStorage) GetTpTableIds(string, string, utils.TPDistinctIds, map[string]string, *utils.Paginator) ([]string, error) { - return nil, nil +func (ms *MongoStorage) GetTpIds() ([]string, error) { + tpidMap := make(map[string]bool) + cols, err := ms.db.CollectionNames() + if err != nil { + return nil, err + } + for _, col := range cols { + if strings.HasPrefix(col, "tp_") { + tpids := make([]string, 0) + if err := ms.db.C(col).Find(nil).Select(bson.M{"tpid": 1}).Distinct("tpid", &tpids); err != nil { + return nil, err + } + for _, tpid := range tpids { + tpidMap[tpid] = true + } + } + } + var tpids []string + for tpid := range tpidMap { + tpids = append(tpids, tpid) + } + return tpids, nil } +func (ms *MongoStorage) GetTpTableIds(tpid, table string, distinct utils.TPDistinctIds, filter map[string]string, pag *utils.Paginator) ([]string, error) { + selectors := bson.M{} + for _, d := range distinct { + selectors[d] = 1 + } + filter["tpid"] = tpid + q := ms.db.C(table).Find(filter) + if pag != nil { + if pag.Limit != nil { + q = q.Limit(*pag.Limit) + } + if pag.Offset != nil { + q = q.Skip(*pag.Offset) + } + } + + iter := q.Select(selectors).Iter() + distinctIds := make(map[string]bool) + item := make(map[string]string) + for iter.Next(item) { + id := "" + last := len(distinct) - 1 + for i, d := range distinct { + if distinctValue, ok := item[d]; ok { + id += distinctValue + } + if i < last { + id += utils.CONCATENATED_KEY_SEP + } + } + distinctIds[id] = true + } + if err := iter.Close(); err != nil { + return nil, err + } + var results []string + for id := range distinctIds { + results = append(results, id) + } + return results, nil +} + func (ms *MongoStorage) GetTpTimings(tpid, tag string) ([]TpTiming, error) { filter := bson.M{ "tpid": tpid, @@ -41,6 +104,7 @@ func (ms *MongoStorage) GetTpTimings(tpid, tag string) ([]TpTiming, error) { err := ms.db.C(colTpTmg).Find(filter).All(&results) return results, err } + func (ms *MongoStorage) GetTpDestinations(tpid, tag string) ([]TpDestination, error) { filter := bson.M{ "tpid": tpid, @@ -52,6 +116,7 @@ func (ms *MongoStorage) GetTpDestinations(tpid, tag string) ([]TpDestination, er err := ms.db.C(colTpDst).Find(filter).All(&results) return results, err } + func (ms *MongoStorage) GetTpRates(tpid, tag string) ([]TpRate, error) { filter := bson.M{ "tpid": tpid, @@ -63,6 +128,7 @@ func (ms *MongoStorage) GetTpRates(tpid, tag string) ([]TpRate, error) { err := ms.db.C(colTpRts).Find(filter).All(&results) return results, err } + func (ms *MongoStorage) GetTpDestinationRates(tpid, tag string, pag *utils.Paginator) ([]TpDestinationRate, error) { filter := bson.M{ "tpid": tpid, @@ -290,9 +356,23 @@ func (ms *MongoStorage) GetTpAccountActions(tp *TpAccountAction) ([]TpAccountAct return results, err } -func (ms *MongoStorage) RemTpData(string, string, ...string) error { - - return nil +func (ms *MongoStorage) RemTpData(table, tpid string, args map[string]string) error { + if len(table) == 0 { // Remove tpid out of all tables + cols, err := ms.db.CollectionNames() + if err != nil { + return err + } + for _, col := range cols { + if strings.HasPrefix(col, "tp_") { + if err := ms.db.C(col).Remove(bson.M{"tpid": tpid}); err != nil { + return err + } + } + } + } + // Remove from a single table + args["tpid"] = tpid + return ms.db.C(table).Remove(args) } func (ms *MongoStorage) SetTpTimings(tps []TpTiming) error { diff --git a/engine/storage_mysql_local_test.go b/engine/storage_mysql_local_test.go index 4f36bf03f..2bcabc1fe 100644 --- a/engine/storage_mysql_local_test.go +++ b/engine/storage_mysql_local_test.go @@ -372,7 +372,7 @@ func TestMySQLRemoveTPData(t *testing.T) { t.Error("Could not store TPTiming") } // Remove Timings - if err := mysqlDb.RemTpData(utils.TBL_TP_TIMINGS, utils.TEST_SQL, tm.TimingId); err != nil { + if err := mysqlDb.RemTpData(utils.TBL_TP_TIMINGS, utils.TEST_SQL, map[string]string{"tag": tm.TimingId}); err != nil { t.Error(err.Error()) } if tmgs, err := mysqlDb.GetTpTimings(utils.TEST_SQL, tm.TimingId); err != nil { @@ -393,7 +393,7 @@ func TestMySQLRemoveTPData(t *testing.T) { t.Error("Could not store TPRatingProfile") } // Remove RatingProfile - if err := mysqlDb.RemTpData(utils.TBL_TP_RATE_PROFILES, rp.TPid, rp.LoadId, rp.Direction, rp.Tenant, rp.Category, rp.Subject); err != nil { + if err := mysqlDb.RemTpData(utils.TBL_TP_RATE_PROFILES, rp.TPid, map[string]string{"loadid": rp.LoadId, "direction": rp.Direction, "tenant": rp.Tenant, "category": rp.Category, "subject": rp.Subject}); err != nil { t.Error(err.Error()) } if rps, err := mysqlDb.GetTpRatingProfiles(&mrp[0]); err != nil { @@ -414,7 +414,7 @@ func TestMySQLRemoveTPData(t *testing.T) { t.Error("Could not create TPAccountActions") } // Remove AccountActions - if err := mysqlDb.RemTpData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.TPid, aa.LoadId, aa.Direction, aa.Tenant, aa.Account); err != nil { + if err := mysqlDb.RemTpData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.TPid, map[string]string{"loadis": aa.LoadId, "direction": aa.Direction, "tenant": aa.Tenant, "account": aa.Account}); err != nil { t.Error(err.Error()) } if aas, err := mysqlDb.GetTpAccountActions(maa); err != nil { @@ -450,7 +450,7 @@ func TestMySQLRemoveTPData(t *testing.T) { t.Error("Could not create TPAccountActions") } // Remove TariffPlan completely - if err := mysqlDb.RemTpData("", utils.TEST_SQL); err != nil { + if err := mysqlDb.RemTpData("", utils.TEST_SQL, nil); err != nil { t.Error(err.Error()) } // Make sure we have removed it diff --git a/engine/storage_psql_local_test.go b/engine/storage_psql_local_test.go index 1926bd74c..0388111ed 100644 --- a/engine/storage_psql_local_test.go +++ b/engine/storage_psql_local_test.go @@ -367,7 +367,7 @@ func TestPSQLRemoveTPData(t *testing.T) { t.Error("Could not store TPTiming") } // Remove Timings - if err := psqlDb.RemTpData(utils.TBL_TP_TIMINGS, utils.TEST_SQL, tm.TimingId); err != nil { + if err := psqlDb.RemTpData(utils.TBL_TP_TIMINGS, utils.TEST_SQL, map[string]string{"tag": tm.TimingId}); err != nil { t.Error(err.Error()) } if tmgs, err := psqlDb.GetTpTimings(utils.TEST_SQL, tm.TimingId); err != nil { @@ -388,7 +388,7 @@ func TestPSQLRemoveTPData(t *testing.T) { t.Error("Could not store TPRatingProfile") } // Remove RatingProfile - if err := psqlDb.RemTpData(utils.TBL_TP_RATE_PROFILES, rp.TPid, rp.LoadId, rp.Direction, rp.Tenant, rp.Category, rp.Subject); err != nil { + if err := psqlDb.RemTpData(utils.TBL_TP_RATE_PROFILES, rp.TPid, map[string]string{"loadid": rp.LoadId, "direction": rp.Direction, "tenant": rp.Tenant, "category": rp.Category, "subject": rp.Subject}); err != nil { t.Error(err.Error()) } if rps, err := psqlDb.GetTpRatingProfiles(&mrp[0]); err != nil { @@ -409,7 +409,7 @@ func TestPSQLRemoveTPData(t *testing.T) { t.Error("Could not create TPAccountActions") } // Remove AccountActions - if err := psqlDb.RemTpData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.TPid, aa.LoadId, aa.Direction, aa.Tenant, aa.Account); err != nil { + if err := psqlDb.RemTpData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.TPid, map[string]string{"loadid": aa.LoadId, "direction": aa.Direction, "tenant": aa.Tenant, "account": aa.Account}); err != nil { t.Error(err.Error()) } if aas, err := psqlDb.GetTpAccountActions(maa); err != nil { @@ -445,7 +445,7 @@ func TestPSQLRemoveTPData(t *testing.T) { t.Error("Could not create TPAccountActions") } // Remove TariffPlan completely - if err := psqlDb.RemTpData("", utils.TEST_SQL); err != nil { + if err := psqlDb.RemTpData("", utils.TEST_SQL, nil); err != nil { t.Error(err.Error()) } // Make sure we have removed it diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 179f8b799..50bac9f5e 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -158,7 +158,7 @@ func (self *SQLStorage) GetTpTableIds(tpid, table string, distinct utils.TPDisti return ids, nil } -func (self *SQLStorage) RemTpData(table, tpid string, args ...string) error { +func (self *SQLStorage) RemTpData(table, tpid string, args map[string]string) error { tx := self.db.Begin() if len(table) == 0 { // Remove tpid out of all tables for _, tblName := range []string{utils.TBL_TP_TIMINGS, utils.TBL_TP_DESTINATIONS, utils.TBL_TP_RATES, utils.TBL_TP_DESTINATION_RATES, utils.TBL_TP_RATING_PLANS, utils.TBL_TP_RATE_PROFILES, @@ -173,15 +173,9 @@ func (self *SQLStorage) RemTpData(table, tpid string, args ...string) error { } // Remove from a single table tx = tx.Table(table).Where("tpid = ?", tpid) - switch table { - default: - tx = tx.Where("tag = ?", args[0]) - case utils.TBL_TP_RATE_PROFILES: - tx = tx.Where("loadid = ?", args[0]).Where("direction = ?", args[1]).Where("tenant = ?", args[2]).Where("category = ?", args[3]).Where("subject = ?", args[4]) - case utils.TBL_TP_ACCOUNT_ACTIONS: - tx = tx.Where("loadid = ?", args[0]).Where("direction = ?", args[1]).Where("tenant = ?", args[2]).Where("account = ?", args[3]) - case utils.TBL_TP_DERIVED_CHARGERS: - tx = tx.Where("loadid = ?", args[0]).Where("direction = ?", args[1]).Where("tenant = ?", args[2]).Where("category = ?", args[3]).Where("account = ?", args[4]).Where("subject = ?", args[5]) + // Compose filters + for key, value := range args { + tx = tx.Where(key+" = ?", value) } if err := tx.Delete(nil).Error; err != nil { tx.Rollback() From 407ccf42c9b1e038ce5c8aed04874e51d9b4cfaf Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 12 Oct 2015 15:57:29 +0300 Subject: [PATCH 3/7] added mongo back as an option in the conf --- cmd/cgr-engine/cgr-engine.go | 2 +- data/conf/samples/cgradmin/cgradmin.json | 7 +++++++ engine/calldesc.go | 4 ++-- engine/storage_mongo.go | 3 ++- engine/storage_utils.go | 16 +++++++++------- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 22bfa2ba7..5cd7aeca2 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -558,7 +558,7 @@ func main() { logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns) if err != nil { // Cannot configure logger database, show stopper - utils.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting!", err)) + utils.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting! (%+v)", err, cfg)) return } defer logDb.Close() diff --git a/data/conf/samples/cgradmin/cgradmin.json b/data/conf/samples/cgradmin/cgradmin.json index 4ea1f0eef..e09f94c20 100644 --- a/data/conf/samples/cgradmin/cgradmin.json +++ b/data/conf/samples/cgradmin/cgradmin.json @@ -10,6 +10,13 @@ "http": ":2080", // HTTP listening address }, +"stor_db": { // database used to store offline tariff plans and CDRs + "db_type": "mongo", // stor database type to use: + "db_host": "127.0.0.1", // the host to connect to + "db_port": 27017, // the port to reach the stordb + "db_name": "cgrates", +}, + "rater": { "enabled": true, // enable Rater service: "pubsubs": "internal", diff --git a/engine/calldesc.go b/engine/calldesc.go index eba37b91d..967fb4582 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -46,11 +46,11 @@ func init() { accountingStorage, _ = NewMapStorage() } else { var err error - ratingStorage, err = NewMongoStorage("127.0.0.1:27017", "cgrates_rating_test", "", "") + ratingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_rating_test", "", "") if err != nil { log.Fatal(err) } - accountingStorage, err = NewMongoStorage("127.0.0.1:27017", "cgrates_accounting_test", "", "") + accountingStorage, err = NewMongoStorage("127.0.0.1", "27017", "cgrates_accounting_test", "", "") if err != nil { log.Fatal(err) } diff --git a/engine/storage_mongo.go b/engine/storage_mongo.go index 49acda18a..149d38da4 100644 --- a/engine/storage_mongo.go +++ b/engine/storage_mongo.go @@ -60,7 +60,8 @@ type MongoStorage struct { db *mgo.Database } -func NewMongoStorage(address, db, user, pass string) (*MongoStorage, error) { +func NewMongoStorage(host, port, db, user, pass string) (*MongoStorage, error) { + address := fmt.Sprintf("%s:%s", host, port) if user != "" && pass != "" { address = fmt.Sprintf("%s:%s@%s", user, pass, address) } diff --git a/engine/storage_utils.go b/engine/storage_utils.go index 6319e92c8..dc19abe43 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -64,11 +64,9 @@ func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler host += ":" + port } d, err = NewRedisStorage(host, db_nb, pass, marshaler) - /* - case utils.MONGO: - d, err = NewMongoStorage(host, port, name, user, pass) - db = d.(AccountingStorage) - */ + case utils.MONGO: + d, err = NewMongoStorage(host, port, name, user, pass) + db = d.(AccountingStorage) default: err = errors.New("unknown db") } @@ -93,9 +91,9 @@ func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string host += ":" + port } d, err = NewRedisStorage(host, db_nb, pass, marshaler) - case utils.MONGO: - d, err = NewMongoStorage(host, port, name, user, pass) */ + case utils.MONGO: + d, err = NewMongoStorage(host, port, name, user, pass) case utils.POSTGRES: d, err = NewPostgresStorage(host, port, name, user, pass, maxConn, maxIdleConn) case utils.MYSQL: @@ -116,6 +114,8 @@ func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler strin d, err = NewPostgresStorage(host, port, name, user, pass, maxConn, maxIdleConn) case utils.MYSQL: d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn) + case utils.MONGO: + d, err = NewMongoStorage(host, port, name, user, pass) default: err = errors.New("unknown db") } @@ -132,6 +132,8 @@ func ConfigureCdrStorage(db_type, host, port, name, user, pass string, maxConn, d, err = NewPostgresStorage(host, port, name, user, pass, maxConn, maxIdleConn) case utils.MYSQL: d, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn) + case utils.MONGO: + d, err = NewMongoStorage(host, port, name, user, pass) default: err = errors.New("unknown db") } From cf067eda4bbbf4e1c5dfff87cef142c894bc7626 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Mon, 12 Oct 2015 20:20:58 +0300 Subject: [PATCH 4/7] tested mongo tp wit cgradmin formal test pending --- cmd/cgr-engine/cgr-engine.go | 2 +- engine/storage_mongo.go | 89 +++++------------- engine/storage_mongo_tp.go | 172 ++++++++++++++++++++++++----------- 3 files changed, 140 insertions(+), 123 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 5cd7aeca2..22bfa2ba7 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -558,7 +558,7 @@ func main() { logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns) if err != nil { // Cannot configure logger database, show stopper - utils.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting! (%+v)", err, cfg)) + utils.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting!", err)) return } defer logDb.Close() diff --git a/engine/storage_mongo.go b/engine/storage_mongo.go index 149d38da4..34344c830 100644 --- a/engine/storage_mongo.go +++ b/engine/storage_mongo.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "strings" - "time" "github.com/cgrates/cgrates/cache2go" "github.com/cgrates/cgrates/utils" @@ -33,26 +32,28 @@ import ( ) const ( - colDst = "destinations" - colAct = "actions" - colApl = "actionplans" - colAtr = "actiontriggers" - colRpl = "ratingplans" - colRpf = "ratingprofiles" - colAcc = "accounts" - colShg = "sharedgroups" - colLcr = "lcrrules" - colDcs = "derivedchargers" - colAls = "aliases" - colStq = "statsqeues" - colPbs = "pubsub" - colUsr = "users" - colCrs = "cdrstats" - colLht = "loadhistory" - colLogCC = "callcostlogs" - colLogAtr = "actiontriggerslogs" - colLogApl = "actionplanlogs" - colLogErr = "errorlogs" + colDst = "destinations" + colAct = "actions" + colApl = "actionplans" + colAtr = "actiontriggers" + colRpl = "ratingplans" + colRpf = "ratingprofiles" + colAcc = "accounts" + colShg = "sharedgroups" + colLcr = "lcrrules" + colDcs = "derivedchargers" + colAls = "aliases" + colStq = "statsqeues" + colPbs = "pubsub" + colUsr = "users" + colCrs = "cdrstats" + colLht = "loadhistory" + colLogCC = "callcostlogs" + colLogAtr = "actiontriggerslogs" + colLogApl = "actionplanlogs" + colLogErr = "errorlogs" + colCdrs = "cdrs" + colRatedCdrs = "ratedcdrs" ) type MongoStorage struct { @@ -123,33 +124,6 @@ func (ms *MongoStorage) Flush(ignore string) (err error) { return nil } -type LogCostEntry struct { - Id string `bson:"_id,omitempty"` - CallCost *CallCost - Source string -} - -type LogTimingEntry struct { - ActionPlan *ActionPlan - Actions Actions - LogTime time.Time - Source string -} - -type LogTriggerEntry struct { - ubId string - ActionTrigger *ActionTrigger - Actions Actions - LogTime time.Time - Source string -} - -type LogErrEntry struct { - Id string `bson:"_id,omitempty"` - ErrStr string - Source string -} - func (ms *MongoStorage) CacheRatingAll() error { return ms.cacheRating(nil, nil, nil, nil, nil, nil, nil) } @@ -1041,22 +1015,3 @@ func (ms *MongoStorage) GetAllCdrStats() (css []*CdrStats, err error) { err = iter.Close() return } - -func (ms *MongoStorage) LogCallCost(cgrid, source string, cc *CallCost) error { - return ms.db.C(colLogCC).Insert(&LogCostEntry{cgrid, cc, source}) -} - -func (ms *MongoStorage) GetCallCostLog(cgrid, source string) (cc *CallCost, err error) { - result := new(LogCostEntry) - err = ms.db.C(colLogCC).Find(bson.M{"_id": cgrid, "source": source}).One(result) - cc = result.CallCost - return -} - -func (ms *MongoStorage) LogActionTrigger(ubId, source string, at *ActionTrigger, as Actions) (err error) { - return ms.db.C(colLogAtr).Insert(&LogTriggerEntry{ubId, at, as, time.Now(), source}) -} - -func (ms *MongoStorage) LogActionPlan(source string, at *ActionPlan, as Actions) (err error) { - return ms.db.C(colLogApl).Insert(&LogTimingEntry{at, as, time.Now(), source}) -} diff --git a/engine/storage_mongo_tp.go b/engine/storage_mongo_tp.go index 4f345d5f1..77a5af084 100644 --- a/engine/storage_mongo_tp.go +++ b/engine/storage_mongo_tp.go @@ -2,32 +2,12 @@ package engine import ( "strings" + "time" "github.com/cgrates/cgrates/utils" "gopkg.in/mgo.v2/bson" ) -const ( - colTpTmg = "tp_timings" - colTpDst = "tp_destinations" - colTpRts = "tp_rates" - colTpDrs = "tp_destinationrates" - colTpRpl = "tp_ratingplans" - colTpRpf = "tp_ratingprofiles" - colTpAct = "tp_actions" - colTpApl = "tp_actionplans" - colTpAtr = "tp_actiontriggers" - colTpAcc = "tp_accounts" - colTpShg = "tp_sharedgroups" - colTpLcr = "tp_lcrrules" - colTpDcs = "tp_derivedchargers" - colTpAls = "tp_aliases" - colTpStq = "tp_statsqeues" - colTpPbs = "tp_pubsub" - colTpUsr = "tp_users" - colTpCrs = "tp_cdrstats" -) - func (ms *MongoStorage) GetTpIds() ([]string, error) { tpidMap := make(map[string]bool) cols, err := ms.db.CollectionNames() @@ -56,8 +36,24 @@ func (ms *MongoStorage) GetTpTableIds(tpid, table string, distinct utils.TPDisti for _, d := range distinct { selectors[d] = 1 } - filter["tpid"] = tpid - q := ms.db.C(table).Find(filter) + findMap := make(map[string]interface{}) + if tpid != "" { + findMap["tpid"] = tpid + } + for k, v := range filter { + findMap[k] = v + } + + if pag != nil && pag.SearchTerm != "" { + var searchItems []bson.M + for _, d := range distinct { + searchItems = append(searchItems, bson.M{d: bson.RegEx{ + Pattern: ".*" + pag.SearchTerm + ".*", + Options: ""}}) + } + findMap["$and"] = []bson.M{bson.M{"$or": searchItems}} + } + q := ms.db.C(table).Find(findMap) if pag != nil { if pag.Limit != nil { q = q.Limit(*pag.Limit) @@ -101,7 +97,7 @@ func (ms *MongoStorage) GetTpTimings(tpid, tag string) ([]TpTiming, error) { filter["tag"] = tag } var results []TpTiming - err := ms.db.C(colTpTmg).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_TIMINGS).Find(filter).All(&results) return results, err } @@ -113,7 +109,7 @@ func (ms *MongoStorage) GetTpDestinations(tpid, tag string) ([]TpDestination, er filter["tag"] = tag } var results []TpDestination - err := ms.db.C(colTpDst).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_DESTINATIONS).Find(filter).All(&results) return results, err } @@ -125,7 +121,7 @@ func (ms *MongoStorage) GetTpRates(tpid, tag string) ([]TpRate, error) { filter["tag"] = tag } var results []TpRate - err := ms.db.C(colTpRts).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_RATES).Find(filter).All(&results) return results, err } @@ -137,7 +133,7 @@ func (ms *MongoStorage) GetTpDestinationRates(tpid, tag string, pag *utils.Pagin filter["tag"] = tag } var results []TpDestinationRate - q := ms.db.C(colTpDrs).Find(filter) + q := ms.db.C(utils.TBL_TP_DESTINATION_RATES).Find(filter) if pag != nil { if pag.Limit != nil { q = q.Limit(*pag.Limit) @@ -158,7 +154,7 @@ func (ms *MongoStorage) GetTpRatingPlans(tpid, tag string, pag *utils.Paginator) filter["tag"] = tag } var results []TpRatingPlan - q := ms.db.C(colTpRpl).Find(filter) + q := ms.db.C(utils.TBL_TP_RATING_PLANS).Find(filter) if pag != nil { if pag.Limit != nil { q = q.Limit(*pag.Limit) @@ -189,7 +185,7 @@ func (ms *MongoStorage) GetTpRatingProfiles(tp *TpRatingProfile) ([]TpRatingProf filter["loadid"] = tp.Loadid } var results []TpRatingProfile - err := ms.db.C(colTpRpf).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_RATE_PROFILES).Find(filter).All(&results) return results, err } @@ -201,7 +197,7 @@ func (ms *MongoStorage) GetTpSharedGroups(tpid, tag string) ([]TpSharedGroup, er filter["tag"] = tag } var results []TpSharedGroup - err := ms.db.C(colTpShg).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_SHARED_GROUPS).Find(filter).All(&results) return results, err } @@ -213,7 +209,7 @@ func (ms *MongoStorage) GetTpCdrStats(tpid, tag string) ([]TpCdrstat, error) { filter["tag"] = tag } var results []TpCdrstat - err := ms.db.C(colTpCrs).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_CDR_STATS).Find(filter).All(&results) return results, err } func (ms *MongoStorage) GetTpLCRs(tp *TpLcrRule) ([]TpLcrRule, error) { @@ -234,7 +230,7 @@ func (ms *MongoStorage) GetTpLCRs(tp *TpLcrRule) ([]TpLcrRule, error) { filter["subject"] = tp.Subject } var results []TpLcrRule - err := ms.db.C(colTpLcr).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_LCRS).Find(filter).All(&results) return results, err } @@ -247,7 +243,7 @@ func (ms *MongoStorage) GetTpUsers(tp *TpUser) ([]TpUser, error) { filter["username"] = tp.UserName } var results []TpUser - err := ms.db.C(colTpUsr).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_USERS).Find(filter).All(&results) return results, err } @@ -272,7 +268,7 @@ func (ms *MongoStorage) GetTpAliases(tp *TpAlias) ([]TpAlias, error) { filter["context"] = tp.Context } var results []TpAlias - err := ms.db.C(colTpAls).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_ALIASES).Find(filter).All(&results) return results, err } @@ -297,7 +293,7 @@ func (ms *MongoStorage) GetTpDerivedChargers(tp *TpDerivedCharger) ([]TpDerivedC filter["loadid"] = tp.Loadid } var results []TpDerivedCharger - err := ms.db.C(colTpDcs).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_DERIVED_CHARGERS).Find(filter).All(&results) return results, err } @@ -309,7 +305,7 @@ func (ms *MongoStorage) GetTpActions(tpid, tag string) ([]TpAction, error) { filter["tag"] = tag } var results []TpAction - err := ms.db.C(colTpAct).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_ACTIONS).Find(filter).All(&results) return results, err } @@ -321,7 +317,7 @@ func (ms *MongoStorage) GetTpActionPlans(tpid, tag string) ([]TpActionPlan, erro filter["tag"] = tag } var results []TpActionPlan - err := ms.db.C(colTpApl).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_ACTION_PLANS).Find(filter).All(&results) return results, err } @@ -333,7 +329,7 @@ func (ms *MongoStorage) GetTpActionTriggers(tpid, tag string) ([]TpActionTrigger filter["tag"] = tag } var results []TpActionTrigger - err := ms.db.C(colTpAtr).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_ACTION_TRIGGERS).Find(filter).All(&results) return results, err } @@ -352,7 +348,7 @@ func (ms *MongoStorage) GetTpAccountActions(tp *TpAccountAction) ([]TpAccountAct filter["loadid"] = tp.Loadid } var results []TpAccountAction - err := ms.db.C(colTpAcc).Find(filter).All(&results) + err := ms.db.C(utils.TBL_TP_ACCOUNT_ACTIONS).Find(filter).All(&results) return results, err } @@ -381,7 +377,7 @@ func (ms *MongoStorage) SetTpTimings(tps []TpTiming) error { } m := make(map[string]bool) - tx := ms.db.C(colTpTmg).Bulk() + tx := ms.db.C(utils.TBL_TP_TIMINGS).Bulk() for _, tp := range tps { if found, _ := m[tp.Tag]; !found { m[tp.Tag] = true @@ -398,7 +394,7 @@ func (ms *MongoStorage) SetTpDestinations(tps []TpDestination) error { } m := make(map[string]bool) - tx := ms.db.C(colTpDst).Bulk() + tx := ms.db.C(utils.TBL_TP_DESTINATIONS).Bulk() for _, tp := range tps { if found, _ := m[tp.Tag]; !found { m[tp.Tag] = true @@ -415,7 +411,7 @@ func (ms *MongoStorage) SetTpRates(tps []TpRate) error { } m := make(map[string]bool) - tx := ms.db.C(colTpRts).Bulk() + tx := ms.db.C(utils.TBL_TP_RATES).Bulk() for _, tp := range tps { if found, _ := m[tp.Tag]; !found { m[tp.Tag] = true @@ -432,7 +428,7 @@ func (ms *MongoStorage) SetTpDestinationRates(tps []TpDestinationRate) error { } m := make(map[string]bool) - tx := ms.db.C(colTpDrs).Bulk() + tx := ms.db.C(utils.TBL_TP_DESTINATION_RATES).Bulk() for _, tp := range tps { if found, _ := m[tp.Tag]; !found { m[tp.Tag] = true @@ -449,7 +445,7 @@ func (ms *MongoStorage) SetTpRatingPlans(tps []TpRatingPlan) error { } m := make(map[string]bool) - tx := ms.db.C(colTpRpl).Bulk() + tx := ms.db.C(utils.TBL_TP_RATING_PLANS).Bulk() for _, tp := range tps { if found, _ := m[tp.Tag]; !found { m[tp.Tag] = true @@ -466,7 +462,7 @@ func (ms *MongoStorage) SetTpRatingProfiles(tps []TpRatingProfile) error { } m := make(map[string]bool) - tx := ms.db.C(colTpRpf).Bulk() + tx := ms.db.C(utils.TBL_TP_RATE_PROFILES).Bulk() for _, tp := range tps { if found, _ := m[tp.GetRatingProfileId()]; !found { m[tp.GetRatingProfileId()] = true @@ -490,7 +486,7 @@ func (ms *MongoStorage) SetTpSharedGroups(tps []TpSharedGroup) error { } m := make(map[string]bool) - tx := ms.db.C(colTpShg).Bulk() + tx := ms.db.C(utils.TBL_TP_SHARED_GROUPS).Bulk() for _, tp := range tps { if found, _ := m[tp.Tag]; !found { m[tp.Tag] = true @@ -507,7 +503,7 @@ func (ms *MongoStorage) SetTpCdrStats(tps []TpCdrstat) error { } m := make(map[string]bool) - tx := ms.db.C(colTpCrs).Bulk() + tx := ms.db.C(utils.TBL_TP_CDR_STATS).Bulk() for _, tp := range tps { if found, _ := m[tp.Tag]; !found { m[tp.Tag] = true @@ -524,7 +520,7 @@ func (ms *MongoStorage) SetTpUsers(tps []TpUser) error { } m := make(map[string]bool) - tx := ms.db.C(colTpUsr).Bulk() + tx := ms.db.C(utils.TBL_TP_USERS).Bulk() for _, tp := range tps { if found, _ := m[tp.GetId()]; !found { m[tp.GetId()] = true @@ -545,7 +541,7 @@ func (ms *MongoStorage) SetTpAliases(tps []TpAlias) error { } m := make(map[string]bool) - tx := ms.db.C(colTpAls).Bulk() + tx := ms.db.C(utils.TBL_TP_ALIASES).Bulk() for _, tp := range tps { if found, _ := m[tp.GetId()]; !found { m[tp.GetId()] = true @@ -569,7 +565,7 @@ func (ms *MongoStorage) SetTpDerivedChargers(tps []TpDerivedCharger) error { } m := make(map[string]bool) - tx := ms.db.C(colTpRpl).Bulk() + tx := ms.db.C(utils.TBL_TP_DERIVED_CHARGERS).Bulk() for _, tp := range tps { if found, _ := m[tp.GetDerivedChargersId()]; !found { m[tp.GetDerivedChargersId()] = true @@ -592,7 +588,7 @@ func (ms *MongoStorage) SetTpLCRs(tps []TpLcrRule) error { } m := make(map[string]bool) - tx := ms.db.C(colTpRpl).Bulk() + tx := ms.db.C(utils.TBL_TP_LCRS).Bulk() for _, tp := range tps { if found, _ := m[tp.GetLcrRuleId()]; !found { m[tp.GetLcrRuleId()] = true @@ -615,7 +611,7 @@ func (ms *MongoStorage) SetTpActions(tps []TpAction) error { } m := make(map[string]bool) - tx := ms.db.C(colTpAct).Bulk() + tx := ms.db.C(utils.TBL_TP_ACTIONS).Bulk() for _, tp := range tps { if found, _ := m[tp.Tag]; !found { m[tp.Tag] = true @@ -632,7 +628,7 @@ func (ms *MongoStorage) SetTpActionPlans(tps []TpActionPlan) error { } m := make(map[string]bool) - tx := ms.db.C(colTpApl).Bulk() + tx := ms.db.C(utils.TBL_TP_ACTION_PLANS).Bulk() for _, tp := range tps { if found, _ := m[tp.Tag]; !found { m[tp.Tag] = true @@ -649,7 +645,7 @@ func (ms *MongoStorage) SetTpActionTriggers(tps []TpActionTrigger) error { } m := make(map[string]bool) - tx := ms.db.C(colTpAtr).Bulk() + tx := ms.db.C(utils.TBL_TP_ACTION_TRIGGERS).Bulk() for _, tp := range tps { if found, _ := m[tp.Tag]; !found { m[tp.Tag] = true @@ -666,7 +662,7 @@ func (ms *MongoStorage) SetTpAccountActions(tps []TpAccountAction) error { } m := make(map[string]bool) - tx := ms.db.C(colTpAcc).Bulk() + tx := ms.db.C(utils.TBL_TP_ACCOUNT_ACTIONS).Bulk() for _, tp := range tps { if found, _ := m[tp.GetAccountActionId()]; !found { m[tp.GetAccountActionId()] = true @@ -681,3 +677,69 @@ func (ms *MongoStorage) SetTpAccountActions(tps []TpAccountAction) error { _, err := tx.Run() return err } + +func (ms *MongoStorage) LogActionTrigger(ubId, source string, at *ActionTrigger, as Actions) (err error) { + return ms.db.C(colLogAtr).Insert(&struct { + ubId string + ActionTrigger *ActionTrigger + Actions Actions + LogTime time.Time + Source string + }{ubId, at, as, time.Now(), source}) +} + +func (ms *MongoStorage) LogActionPlan(source string, at *ActionPlan, as Actions) (err error) { + return ms.db.C(colLogApl).Insert(&struct { + ActionPlan *ActionPlan + Actions Actions + LogTime time.Time + Source string + }{at, as, time.Now(), source}) +} + +func (ms *MongoStorage) LogCallCost(cgrid, source, runid string, cc *CallCost) error { + return ms.db.C(colLogCC).Insert(&struct { + Id string `bson:"id,omitempty"` + Source string + Runid string `bson:"runid,omitempty"` + CallCost *CallCost + }{cgrid, source, runid, cc}) +} + +func (ms *MongoStorage) GetCallCostLog(cgrid, source, runid string) (cc *CallCost, err error) { + result := &struct { + Id string `bson:"id,omitempty"` + Source string + Runid string `bson:"runid,omitempty"` + CallCost *CallCost + }{} + err = ms.db.C(colLogCC).Find(bson.M{"_id": cgrid, "source": source}).One(result) + cc = result.CallCost + return +} + +func (ms *MongoStorage) SetCdr(cdr *StoredCdr) error { + return ms.db.C(colCdrs).Insert(cdr) +} + +func (ms *MongoStorage) SetRatedCdr(storedCdr *StoredCdr) error { + return ms.db.C(colRatedCdrs).Insert(storedCdr) +} + +// Remove CDR data out of all CDR tables based on their cgrid +func (ms *MongoStorage) RemStoredCdrs(cgrIds []string) error { + if len(cgrIds) == 0 { + return nil + } + + for _, col := range []string{colCdrs, colRatedCdrs, colLogCC} { + if err := ms.db.C(col).Update(bson.M{"cgrid": bson.M{"$in": cgrIds}}, map[string]interface{}{"deleted_at": time.Now()}); err != nil { + return err + } + } + return nil +} + +func (ms *MongoStorage) GetStoredCdrs(qryFltr *utils.CdrsFilter) ([]*StoredCdr, int64, error) { + return nil, 0, utils.ErrNotImplemented +} From ff0e33d3b7c8553cc7493b0614508dd2e25546dd Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 13 Oct 2015 09:41:48 +0300 Subject: [PATCH 5/7] preparing local test --- data/docker/devel/start.sh | 3 +++ engine/calldesc.go | 2 +- engine/storage_mysql_local_test.go | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/data/docker/devel/start.sh b/data/docker/devel/start.sh index 0a4b410a9..75df8c736 100755 --- a/data/docker/devel/start.sh +++ b/data/docker/devel/start.sh @@ -19,6 +19,9 @@ cd /usr/share/cgrates/storage/mysql && ./setup_cgr_db.sh root CGRateS.org # setup postgres cd /usr/share/cgrates/storage/postgres && ./setup_cgr_db.sh +# create cgrates user for mongo +mongo --eval 'db.createUser({"user":"cgrates", "pwd":"CGRateS.org", "roles":[{role: "userAdminAnyDatabase", db: "admin"}]})' admin + #env vars export GOROOT=/root/go; export GOPATH=/root/code; export PATH=$GOROOT/bin:$GOPATH/bin:$PATH export GO15VENDOREXPERIMENT=1 diff --git a/engine/calldesc.go b/engine/calldesc.go index 967fb4582..0b69069bb 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -37,7 +37,7 @@ const ( RECURSION_MAX_DEPTH = 3 MIN_PREFIX_MATCH = 1 FALLBACK_SUBJECT = utils.ANY - DEBUG = false + DEBUG = true ) func init() { diff --git a/engine/storage_mysql_local_test.go b/engine/storage_mysql_local_test.go index 2bcabc1fe..5f8a974e5 100644 --- a/engine/storage_mysql_local_test.go +++ b/engine/storage_mysql_local_test.go @@ -414,7 +414,7 @@ func TestMySQLRemoveTPData(t *testing.T) { t.Error("Could not create TPAccountActions") } // Remove AccountActions - if err := mysqlDb.RemTpData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.TPid, map[string]string{"loadis": aa.LoadId, "direction": aa.Direction, "tenant": aa.Tenant, "account": aa.Account}); err != nil { + if err := mysqlDb.RemTpData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.TPid, map[string]string{"loadid": aa.LoadId, "direction": aa.Direction, "tenant": aa.Tenant, "account": aa.Account}); err != nil { t.Error(err.Error()) } if aas, err := mysqlDb.GetTpAccountActions(maa); err != nil { From 17603c4c51c5470f9fa97032994d1356f72c69a2 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 13 Oct 2015 16:29:59 +0300 Subject: [PATCH 6/7] added mongo local tests --- apier/v2/cdrs_mysql_local_test.go | 15 +- apier/v2/cdrs_psql_local_test.go | 15 +- cdrc/cdrc_local_test.go | 6 +- engine/loader_local_test.go | 7 +- engine/storage_mongo.go | 131 ++- engine/storage_mongo_local_test.go | 975 +++++++++++++++++++++++ engine/storage_mongo_tp.go | 20 +- engine/storage_mysql.go | 2 +- engine/storage_mysql_local_test.go | 5 +- engine/storage_postgres.go | 2 +- engine/storage_psql_local_test.go | 5 +- engine/storage_utils.go | 16 +- general_tests/multiplecdrc_local_test.go | 6 +- 13 files changed, 1147 insertions(+), 58 deletions(-) create mode 100644 engine/storage_mongo_local_test.go diff --git a/apier/v2/cdrs_mysql_local_test.go b/apier/v2/cdrs_mysql_local_test.go index fb155b48a..de3049542 100644 --- a/apier/v2/cdrs_mysql_local_test.go +++ b/apier/v2/cdrs_mysql_local_test.go @@ -20,14 +20,15 @@ package v2 import ( "flag" - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" "net/rpc" "net/rpc/jsonrpc" "path" "testing" "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" ) var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args @@ -72,13 +73,11 @@ func TestV2CdrsMysqlInjectUnratedCdr(t *testing.T) { if !*testLocal { return } - var mysqlDb *engine.MySQLStorage - if d, err := engine.NewMySQLStorage(cdrsCfg.StorDBHost, cdrsCfg.StorDBPort, cdrsCfg.StorDBName, cdrsCfg.StorDBUser, cdrsCfg.StorDBPass, - cdrsCfg.StorDBMaxOpenConns, cdrsCfg.StorDBMaxIdleConns); err != nil { + mysqlDb, err := engine.NewMySQLStorage(cdrsCfg.StorDBHost, cdrsCfg.StorDBPort, cdrsCfg.StorDBName, cdrsCfg.StorDBUser, cdrsCfg.StorDBPass, + cdrsCfg.StorDBMaxOpenConns, cdrsCfg.StorDBMaxIdleConns) + if err != nil { t.Error("Error on opening database connection: ", err) return - } else { - mysqlDb = d.(*engine.MySQLStorage) } strCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()), TOR: utils.VOICE, AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: utils.META_RATED, diff --git a/apier/v2/cdrs_psql_local_test.go b/apier/v2/cdrs_psql_local_test.go index 80c593f0a..821a406ed 100644 --- a/apier/v2/cdrs_psql_local_test.go +++ b/apier/v2/cdrs_psql_local_test.go @@ -19,15 +19,16 @@ along with this program. If not, see package v2 import ( - "github.com/cgrates/cgrates/config" - "github.com/cgrates/cgrates/engine" - "github.com/cgrates/cgrates/utils" "net/rpc" "net/rpc/jsonrpc" "os/exec" "path" "testing" "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" ) var cdrsPsqlCfgPath string @@ -69,13 +70,11 @@ func TestV2CdrsPsqlInjectUnratedCdr(t *testing.T) { if !*testLocal { return } - var psqlDb *engine.PostgresStorage - if d, err := engine.NewPostgresStorage(cdrsPsqlCfg.StorDBHost, cdrsPsqlCfg.StorDBPort, cdrsPsqlCfg.StorDBName, cdrsPsqlCfg.StorDBUser, cdrsPsqlCfg.StorDBPass, - cdrsPsqlCfg.StorDBMaxOpenConns, cdrsPsqlCfg.StorDBMaxIdleConns); err != nil { + psqlDb, err := engine.NewPostgresStorage(cdrsPsqlCfg.StorDBHost, cdrsPsqlCfg.StorDBPort, cdrsPsqlCfg.StorDBName, cdrsPsqlCfg.StorDBUser, cdrsPsqlCfg.StorDBPass, + cdrsPsqlCfg.StorDBMaxOpenConns, cdrsPsqlCfg.StorDBMaxIdleConns) + if err != nil { t.Error("Error on opening database connection: ", err) return - } else { - psqlDb = d.(*engine.PostgresStorage) } strCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()), TOR: utils.VOICE, AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: utils.META_RATED, diff --git a/cdrc/cdrc_local_test.go b/cdrc/cdrc_local_test.go index 5e0521d73..fb3737c32 100644 --- a/cdrc/cdrc_local_test.go +++ b/cdrc/cdrc_local_test.go @@ -111,11 +111,9 @@ func TestCsvLclEmptyTables(t *testing.T) { if *storDbType != utils.MYSQL { t.Fatal("Unsupported storDbType") } - var mysql *engine.MySQLStorage - if d, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns); err != nil { + mysql, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns) + if err != nil { t.Fatal("Error on opening database connection: ", err) - } else { - mysql = d.(*engine.MySQLStorage) } for _, scriptName := range []string{utils.CREATE_CDRS_TABLES_SQL, utils.CREATE_TARIFFPLAN_TABLES_SQL} { if err := mysql.CreateTablesFromScript(path.Join(*dataDir, "storage", *storDbType, scriptName)); err != nil { diff --git a/engine/loader_local_test.go b/engine/loader_local_test.go index ee72d937f..f502f1740 100644 --- a/engine/loader_local_test.go +++ b/engine/loader_local_test.go @@ -90,13 +90,12 @@ func TestCreateStorTpTables(t *testing.T) { if !*testLocal { return } - var db *MySQLStorage - if d, err := NewMySQLStorage(lCfg.StorDBHost, lCfg.StorDBPort, lCfg.StorDBName, lCfg.StorDBUser, lCfg.StorDBPass, lCfg.StorDBMaxOpenConns, lCfg.StorDBMaxIdleConns); err != nil { + db, err := NewMySQLStorage(lCfg.StorDBHost, lCfg.StorDBPort, lCfg.StorDBName, lCfg.StorDBUser, lCfg.StorDBPass, lCfg.StorDBMaxOpenConns, lCfg.StorDBMaxIdleConns) + if err != nil { t.Error("Error on opening database connection: ", err) return } else { - db = d.(*MySQLStorage) - storDb = d.(LoadStorage) + storDb = db } // Creating the table serves also as reset since there is a drop prior to create if err := db.CreateTablesFromScript(path.Join(*dataDir, "storage", "mysql", utils.CREATE_TARIFFPLAN_TABLES_SQL)); err != nil { diff --git a/engine/storage_mongo.go b/engine/storage_mongo.go index 34344c830..b86b3277a 100644 --- a/engine/storage_mongo.go +++ b/engine/storage_mongo.go @@ -79,10 +79,9 @@ func NewMongoStorage(host, port, db, user, pass string) (*MongoStorage, error) { Background: false, // Build index in background and return immediately Sparse: false, // Only index documents containing the Key fields } - keyCollections := []string{colAct, colApl, colAtr, colDcs, colAls, colUsr, colLcr, colLht} - for _, col := range keyCollections { - err = ndb.C(col).EnsureIndex(index) - if err != nil { + collections := []string{colAct, colApl, colAtr, colDcs, colAls, colUsr, colLcr, colLht} + for _, col := range collections { + if err = ndb.C(col).EnsureIndex(index); err != nil { return nil, err } } @@ -93,10 +92,126 @@ func NewMongoStorage(host, port, db, user, pass string) (*MongoStorage, error) { Background: false, Sparse: false, } - idCollections := []string{colDst, colRpf, colRpl, colDst, colShg, colAcc, colCrs} - for _, col := range idCollections { - err = ndb.C(col).EnsureIndex(index) - if err != nil { + collections = []string{colDst, colRpf, colRpl, colDst, colShg, colAcc, colCrs} + for _, col := range collections { + if err = ndb.C(col).EnsureIndex(index); err != nil { + return nil, err + } + } + index = mgo.Index{ + Key: []string{"tpid", "tag"}, + Unique: true, + DropDups: false, + Background: false, + Sparse: false, + } + collections = []string{utils.TBL_TP_TIMINGS, utils.TBL_TP_DESTINATIONS, utils.TBL_TP_DESTINATION_RATES, utils.TBL_TP_RATING_PLANS, utils.TBL_TP_SHARED_GROUPS, utils.TBL_TP_CDR_STATS, utils.TBL_TP_ACTIONS, utils.TBL_TP_ACTION_PLANS, utils.TBL_TP_ACTION_TRIGGERS} + for _, col := range collections { + if err = ndb.C(col).EnsureIndex(index); err != nil { + return nil, err + } + } + index = mgo.Index{ + Key: []string{"tpid", "direction", "tenant", "category", "subject", "loadid"}, + Unique: true, + DropDups: false, + Background: false, + Sparse: false, + } + collections = []string{utils.TBL_TP_RATE_PROFILES} + for _, col := range collections { + if err = ndb.C(col).EnsureIndex(index); err != nil { + return nil, err + } + } + index = mgo.Index{ + Key: []string{"tpid", "direction", "tenant", "category", "account", "subject"}, + Unique: true, + DropDups: false, + Background: false, + Sparse: false, + } + collections = []string{utils.TBL_TP_LCRS} + for _, col := range collections { + if err = ndb.C(col).EnsureIndex(index); err != nil { + return nil, err + } + } + index = mgo.Index{ + Key: []string{"tpid", "tenant", "username"}, + Unique: true, + DropDups: false, + Background: false, + Sparse: false, + } + collections = []string{utils.TBL_TP_USERS} + for _, col := range collections { + if err = ndb.C(col).EnsureIndex(index); err != nil { + return nil, err + } + } + index = mgo.Index{ + Key: []string{"tpid", "direction", "tenant", "category", "account", "subject", "context"}, + Unique: true, + DropDups: false, + Background: false, + Sparse: false, + } + collections = []string{utils.TBL_TP_LCRS} + for _, col := range collections { + if err = ndb.C(col).EnsureIndex(index); err != nil { + return nil, err + } + } + index = mgo.Index{ + Key: []string{"tpid", "direction", "tenant", "category", "subject", "account", "loadid"}, + Unique: true, + DropDups: false, + Background: false, + Sparse: false, + } + collections = []string{utils.TBL_TP_DERIVED_CHARGERS} + for _, col := range collections { + if err = ndb.C(col).EnsureIndex(index); err != nil { + return nil, err + } + } + index = mgo.Index{ + Key: []string{"tpid", "direction", "tenant", "account", "loadid"}, + Unique: true, + DropDups: false, + Background: false, + Sparse: false, + } + collections = []string{utils.TBL_TP_DERIVED_CHARGERS} + for _, col := range collections { + if err = ndb.C(col).EnsureIndex(index); err != nil { + return nil, err + } + } + index = mgo.Index{ + Key: []string{"id", "source", "runid"}, + Unique: true, + DropDups: false, + Background: false, + Sparse: false, + } + collections = []string{colLogCC} + for _, col := range collections { + if err = ndb.C(col).EnsureIndex(index); err != nil { + return nil, err + } + } + index = mgo.Index{ + Key: []string{"cgrid"}, + Unique: true, + DropDups: false, + Background: false, + Sparse: false, + } + collections = []string{colCdrs, colRatedCdrs} + for _, col := range collections { + if err = ndb.C(col).EnsureIndex(index); err != nil { return nil, err } } diff --git a/engine/storage_mongo_local_test.go b/engine/storage_mongo_local_test.go new file mode 100644 index 000000000..066234708 --- /dev/null +++ b/engine/storage_mongo_local_test.go @@ -0,0 +1,975 @@ +/* +Rating system designed to be used in VoIP Carriers World +Copyright (C) 2012-2015 ITsysCOM + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package engine + +import ( + "reflect" + "testing" + "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/utils" +) + +var mongoDb *MongoStorage + +func TestMongoCreateTables(t *testing.T) { + if !*testLocal { + return + } + cgrConfig, _ := config.NewDefaultCGRConfig() + var err error + if mongoDb, err = NewMongoStorage("localhost", "27017", cgrConfig.StorDBName, cgrConfig.StorDBUser, cgrConfig.StorDBPass); err != nil { + t.Error("Error on opening database connection: ", err) + } +} + +func TestMongoSetGetTPTiming(t *testing.T) { + if !*testLocal { + return + } + tm := TpTiming{Tpid: utils.TEST_SQL, Tag: "ALWAYS", Time: "00:00:00"} + if err := mongoDb.SetTpTimings([]TpTiming{tm}); err != nil { + t.Error(err.Error()) + } + if tmgs, err := mongoDb.GetTpTimings(utils.TEST_SQL, tm.Tag); err != nil { + t.Error(err.Error()) + } else if !modelEqual(tm, tmgs[0]) { + t.Errorf("Expecting: %+v, received: %+v", tm, tmgs[0]) + } + // Update + tm.Time = "00:00:01" + if err := mongoDb.SetTpTimings([]TpTiming{tm}); err != nil { + t.Error(err.Error()) + } + if tmgs, err := mongoDb.GetTpTimings(utils.TEST_SQL, tm.Tag); err != nil { + t.Error(err.Error()) + } else if !modelEqual(tm, tmgs[0]) { + t.Errorf("Expecting: %+v, received: %+v", tm, tmgs[0]) + } +} + +func TestMongoSetGetTPDestination(t *testing.T) { + if !*testLocal { + return + } + dst := []TpDestination{ + TpDestination{Tpid: utils.TEST_SQL, Tag: utils.TEST_SQL, Prefix: "+49"}, + TpDestination{Tpid: utils.TEST_SQL, Tag: utils.TEST_SQL, Prefix: "+49151"}, + TpDestination{Tpid: utils.TEST_SQL, Tag: utils.TEST_SQL, Prefix: "+49176"}, + } + if err := mongoDb.SetTpDestinations(dst); err != nil { + t.Error(err.Error()) + } + storData, err := mongoDb.GetTpDestinations(utils.TEST_SQL, utils.TEST_SQL) + dsts, err := TpDestinations(storData).GetDestinations() + expected := &Destination{Id: utils.TEST_SQL, Prefixes: []string{"+49", "+49151", "+49176"}} + if err != nil { + t.Error(err.Error()) + } else if !modelEqual(*expected, *dsts[utils.TEST_SQL]) { + t.Errorf("Expecting: %+v, received: %+v", expected, dsts[utils.TEST_SQL]) + } +} + +func TestMongoSetGetTPRates(t *testing.T) { + if !*testLocal { + return + } + RT_ID := "RT_1" + rtSlots := []*utils.RateSlot{ + &utils.RateSlot{ConnectFee: 0.02, Rate: 0.01, RateUnit: "60s", RateIncrement: "60s", GroupIntervalStart: "0s"}, + &utils.RateSlot{ConnectFee: 0.00, Rate: 0.005, RateUnit: "60s", RateIncrement: "1s", GroupIntervalStart: "60s"}, + } + for _, rs := range rtSlots { + rs.SetDurations() + } + rates := &utils.TPRate{ + TPid: utils.TEST_SQL, + RateId: RT_ID, + RateSlots: rtSlots, + } + mRates := APItoModelRate(rates) + if err := mongoDb.SetTpRates(mRates); err != nil { + t.Error(err.Error()) + } + if rts, err := mongoDb.GetTpRates(utils.TEST_SQL, RT_ID); err != nil { + t.Error(err.Error()) + } else if !modelEqual(mRates[0], rts[0]) { + t.Errorf("Expecting: %+v, received: %+v", mRates, rts) + } +} + +func TestMongoSetGetTPDestinationRates(t *testing.T) { + if !*testLocal { + return + } + DR_ID := "DR_1" + dr := &utils.DestinationRate{DestinationId: "DST_1", RateId: "RT_1", RoundingMethod: "*up", RoundingDecimals: 4} + + eDrs := &utils.TPDestinationRate{TPid: utils.TEST_SQL, DestinationRateId: DR_ID, DestinationRates: []*utils.DestinationRate{dr}} + mdrs := APItoModelDestinationRate(eDrs) + if err := mongoDb.SetTpDestinationRates(mdrs); err != nil { + t.Error(err.Error()) + } + if drs, err := mongoDb.GetTpDestinationRates(utils.TEST_SQL, DR_ID, nil); err != nil { + t.Error(err.Error()) + } else if !modelEqual(mdrs[0], drs[0]) { + t.Errorf("Expecting: %+v, received: %+v", mdrs, drs) + } +} + +func TestMongoSetGetTPRatingPlans(t *testing.T) { + if !*testLocal { + return + } + RP_ID := "RP_1" + rbBinding := &utils.TPRatingPlanBinding{DestinationRatesId: "DR_1", TimingId: "TM_1", Weight: 10.0} + rp := &utils.TPRatingPlan{ + TPid: utils.TEST_SQL, + RatingPlanId: RP_ID, + RatingPlanBindings: []*utils.TPRatingPlanBinding{rbBinding}, + } + mrp := APItoModelRatingPlan(rp) + if err := mongoDb.SetTpRatingPlans(mrp); err != nil { + t.Error(err.Error()) + } + if drps, err := mongoDb.GetTpRatingPlans(utils.TEST_SQL, RP_ID, nil); err != nil { + t.Error(err.Error()) + } else if !modelEqual(mrp[0], drps[0]) { + t.Errorf("Expecting: %+v, received: %+v", mrp, drps) + } +} + +func TestMongoSetGetTPRatingProfiles(t *testing.T) { + if !*testLocal { + return + } + ras := []*utils.TPRatingActivation{&utils.TPRatingActivation{ActivationTime: "2012-01-01T00:00:00Z", RatingPlanId: "RP_1"}} + rp := &utils.TPRatingProfile{TPid: utils.TEST_SQL, LoadId: utils.TEST_SQL, Tenant: "cgrates.org", Category: "call", Direction: "*out", Subject: "*any", RatingPlanActivations: ras} + mrp := APItoModelRatingProfile(rp) + if err := mongoDb.SetTpRatingProfiles(mrp); err != nil { + t.Error(err.Error()) + } + if rps, err := mongoDb.GetTpRatingProfiles(&mrp[0]); err != nil { + t.Error(err.Error()) + } else if !modelEqual(mrp[0], rps[0]) { + t.Errorf("Expecting: %+v, received: %+v", mrp, rps) + } + +} + +func TestMongoSetGetTPSharedGroups(t *testing.T) { + if !*testLocal { + return + } + SG_ID := "SG_1" + tpSgs := &utils.TPSharedGroups{ + TPid: utils.TEST_SQL, + SharedGroupsId: SG_ID, + SharedGroups: []*utils.TPSharedGroup{ + &utils.TPSharedGroup{Account: "dan", Strategy: "*lowest_first", RatingSubject: "lowest_rates"}, + }, + } + mSgs := APItoModelSharedGroup(tpSgs) + if err := mongoDb.SetTpSharedGroups(mSgs); err != nil { + t.Error(err.Error()) + } + if sgs, err := mongoDb.GetTpSharedGroups(utils.TEST_SQL, SG_ID); err != nil { + t.Error(err.Error()) + } else if !modelEqual(mSgs[0], sgs[0]) { + t.Errorf("Expecting: %+v, received: %+v", mSgs, sgs) + } +} + +func TestMongoSetGetTPCdrStats(t *testing.T) { + if !*testLocal { + return + } + CS_ID := "CDRSTATS_1" + setCS := &utils.TPCdrStats{ + TPid: utils.TEST_SQL, + CdrStatsId: CS_ID, + CdrStats: []*utils.TPCdrStat{ + &utils.TPCdrStat{QueueLength: "10", TimeWindow: "10m", Metrics: "ASR", Tenants: "cgrates.org", Categories: "call"}, + }, + } + mcs := APItoModelCdrStat(setCS) + if err := mongoDb.SetTpCdrStats(mcs); err != nil { + t.Error(err.Error()) + } + if cs, err := mongoDb.GetTpCdrStats(utils.TEST_SQL, CS_ID); err != nil { + t.Error(err.Error()) + } else if !modelEqual(mcs[0], cs[0]) { + t.Errorf("Expecting: %+v, received: %+v", mcs, cs) + } +} + +func TestMongoSetGetTPDerivedChargers(t *testing.T) { + if !*testLocal { + return + } + dc := &utils.TPDerivedCharger{RunId: utils.DEFAULT_RUNID, ReqTypeField: "^" + utils.META_PREPAID, AccountField: "^rif", SubjectField: "^rif", + UsageField: "cgr_duration", SupplierField: "^supplier1"} + dcs := &utils.TPDerivedChargers{TPid: utils.TEST_SQL, Direction: utils.OUT, Tenant: "cgrates.org", Category: "call", Account: "dan", Subject: "dan", DerivedChargers: []*utils.TPDerivedCharger{dc}} + + mdcs := APItoModelDerivedCharger(dcs) + if err := mongoDb.SetTpDerivedChargers(mdcs); err != nil { + t.Error(err.Error()) + } + if rDCs, err := mongoDb.GetTpDerivedChargers(&mdcs[0]); err != nil { + t.Error(err.Error()) + } else if !modelEqual(mdcs[0], rDCs[0]) { + t.Errorf("Expecting: %+v, received: %+v", mdcs, rDCs) + } +} + +func TestMongoSetGetTPActions(t *testing.T) { + if !*testLocal { + return + } + ACTS_ID := "PREPAID_10" + acts := []*utils.TPAction{ + &utils.TPAction{Identifier: "*topup_reset", BalanceType: "*monetary", Direction: "*out", Units: 10, ExpiryTime: "*unlimited", + DestinationIds: "*any", BalanceWeight: 10, Weight: 10}} + tpActions := &utils.TPActions{TPid: utils.TEST_SQL, ActionsId: ACTS_ID, Actions: acts} + mas := APItoModelAction(tpActions) + if err := mongoDb.SetTpActions(mas); err != nil { + t.Error(err.Error()) + } + if rTpActs, err := mongoDb.GetTpActions(utils.TEST_SQL, ACTS_ID); err != nil { + t.Error(err.Error()) + } else if !modelEqual(mas[0], rTpActs[0]) { + t.Errorf("Expecting: %+v, received: %+v", mas, rTpActs) + } +} + +func TestMongoTPActionTimings(t *testing.T) { + if !*testLocal { + return + } + AP_ID := "AP_1" + ap := &utils.TPActionPlan{ + TPid: utils.TEST_SQL, + ActionPlanId: AP_ID, + ActionPlan: []*utils.TPActionTiming{&utils.TPActionTiming{ActionsId: "ACTS_1", TimingId: "TM_1", Weight: 10.0}}, + } + maps := APItoModelActionPlan(ap) + if err := mongoDb.SetTpActionPlans(maps); err != nil { + t.Error(err.Error()) + } + if rAP, err := mongoDb.GetTpActionPlans(utils.TEST_SQL, AP_ID); err != nil { + t.Error(err.Error()) + } else if !modelEqual(maps[0], rAP[0]) { + t.Errorf("Expecting: %+v, received: %+v", maps, rAP) + } +} + +func TestMongoSetGetTPActionTriggers(t *testing.T) { + if !*testLocal { + return + } + atrg := &utils.TPActionTrigger{ + Id: "MY_FIRST_ATGR", + BalanceType: "*monetary", + BalanceDirection: "*out", + ThresholdType: "*min_balance", + ThresholdValue: 2.0, + Recurrent: true, + BalanceDestinationIds: "*any", + Weight: 10.0, + ActionsId: "LOG_BALANCE", + } + atrgs := &utils.TPActionTriggers{ + TPid: utils.TEST_SQL, + ActionTriggersId: utils.TEST_SQL, + ActionTriggers: []*utils.TPActionTrigger{atrg}, + } + matrg := APItoModelActionTrigger(atrgs) + if err := mongoDb.SetTpActionTriggers(matrg); err != nil { + t.Error("Unexpected error: ", err.Error()) + } + if rcvMpAtrgs, err := mongoDb.GetTpActionTriggers(utils.TEST_SQL, utils.TEST_SQL); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if !modelEqual(matrg[0], rcvMpAtrgs[0]) { + t.Errorf("Expecting: %v, received: %v", matrg, rcvMpAtrgs) + } +} + +func TestMongoSetGetTpAccountActions(t *testing.T) { + if !*testLocal { + return + } + aa := &utils.TPAccountActions{TPid: utils.TEST_SQL, Tenant: "cgrates.org", Account: "1001", + Direction: "*out", ActionPlanId: "PREPAID_10", ActionTriggersId: "STANDARD_TRIGGERS"} + maa := APItoModelAccountAction(aa) + if err := mongoDb.SetTpAccountActions([]TpAccountAction{*maa}); err != nil { + t.Error(err.Error()) + } + if aas, err := mongoDb.GetTpAccountActions(maa); err != nil { + t.Error(err.Error()) + } else if !modelEqual(*maa, aas[0]) { + t.Errorf("Expecting: %+v, received: %+v", maa, aas) + } +} + +func TestMongoGetTPIds(t *testing.T) { + if !*testLocal { + return + } + eTPIds := []string{utils.TEST_SQL} + if tpIds, err := mongoDb.GetTpIds(); err != nil { + t.Error(err.Error()) + } else if !reflect.DeepEqual(eTPIds, tpIds) { + t.Errorf("Expecting: %+v, received: %+v", eTPIds, tpIds) + } +} + +func TestMongoRemoveTPData(t *testing.T) { + if !*testLocal { + return + } + // Create Timings + tm := &utils.ApierTPTiming{TPid: utils.TEST_SQL, TimingId: "ALWAYS", Time: "00:00:00"} + tms := APItoModelTiming(tm) + if err := mongoDb.SetTpTimings([]TpTiming{*tms}); err != nil { + t.Error(err.Error()) + } + if tmgs, err := mongoDb.GetTpTimings(utils.TEST_SQL, tm.TimingId); err != nil { + t.Error(err.Error()) + } else if len(tmgs) == 0 { + t.Error("Could not store TPTiming") + } + // Remove Timings + if err := mongoDb.RemTpData(utils.TBL_TP_TIMINGS, utils.TEST_SQL, map[string]string{"tag": tm.TimingId}); err != nil { + t.Error(err.Error()) + } + if tmgs, err := mongoDb.GetTpTimings(utils.TEST_SQL, tm.TimingId); err != nil { + t.Error(err) + } else if len(tmgs) != 0 { + t.Errorf("Timings should be empty, got instead: %+v", tmgs) + } + // Create RatingProfile + ras := []*utils.TPRatingActivation{&utils.TPRatingActivation{ActivationTime: "2012-01-01T00:00:00Z", RatingPlanId: "RETAIL1"}} + rp := &utils.TPRatingProfile{TPid: utils.TEST_SQL, LoadId: utils.TEST_SQL, Tenant: "cgrates.org", Category: "call", Direction: "*out", Subject: "*any", RatingPlanActivations: ras} + mrp := APItoModelRatingProfile(rp) + if err := mongoDb.SetTpRatingProfiles(mrp); err != nil { + t.Error(err.Error()) + } + if rps, err := mongoDb.GetTpRatingProfiles(&mrp[0]); err != nil { + t.Error(err.Error()) + } else if len(rps) == 0 { + t.Error("Could not store TPRatingProfile") + } + // Remove RatingProfile + if err := mongoDb.RemTpData(utils.TBL_TP_RATE_PROFILES, rp.TPid, map[string]string{"loadid": rp.LoadId, "direction": rp.Direction, "tenant": rp.Tenant, "category": rp.Category, "subject": rp.Subject}); err != nil { + t.Error(err.Error()) + } + if rps, err := mongoDb.GetTpRatingProfiles(&mrp[0]); err != nil { + t.Error(err) + } else if len(rps) != 0 { + t.Errorf("RatingProfiles different than 0: %+v", rps) + } + // Create AccountActions + aa := &utils.TPAccountActions{TPid: utils.TEST_SQL, LoadId: utils.TEST_SQL, Tenant: "cgrates.org", Account: "1001", + Direction: "*out", ActionPlanId: "PREPAID_10", ActionTriggersId: "STANDARD_TRIGGERS"} + maa := APItoModelAccountAction(aa) + if err := mongoDb.SetTpAccountActions([]TpAccountAction{*maa}); err != nil { + t.Error(err.Error()) + } + if aas, err := mongoDb.GetTpAccountActions(maa); err != nil { + t.Error(err.Error()) + } else if len(aas) == 0 { + t.Error("Could not create TPAccountActions") + } + // Remove AccountActions + if err := mongoDb.RemTpData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.TPid, map[string]string{"loadid": aa.LoadId, "direction": aa.Direction, "tenant": aa.Tenant, "account": aa.Account}); err != nil { + t.Error(err.Error()) + } + if aas, err := mongoDb.GetTpAccountActions(maa); err != nil { + t.Error(err) + } else if len(aas) != 0 { + t.Errorf("Non empty account actions: %+v", aas) + } + // Create again so we can test complete TP removal + if err := mongoDb.SetTpTimings([]TpTiming{*tms}); err != nil { + t.Error(err.Error()) + } + if tmgs, err := mongoDb.GetTpTimings(utils.TEST_SQL, tm.TimingId); err != nil { + t.Error(err.Error()) + } else if len(tmgs) == 0 { + t.Error("Could not store TPTiming") + } + // Create RatingProfile + if err := mongoDb.SetTpRatingProfiles(mrp); err != nil { + t.Error(err.Error()) + } + if rps, err := mongoDb.GetTpRatingProfiles(&mrp[0]); err != nil { + t.Error(err.Error()) + } else if len(rps) == 0 { + t.Error("Could not store TPRatingProfile") + } + // Create AccountActions + if err := mongoDb.SetTpAccountActions([]TpAccountAction{*maa}); err != nil { + t.Error(err.Error()) + } + if aas, err := mongoDb.GetTpAccountActions(maa); err != nil { + t.Error(err.Error()) + } else if len(aas) == 0 { + t.Error("Could not create TPAccountActions") + } + // Remove TariffPlan completely + if err := mongoDb.RemTpData("", utils.TEST_SQL, nil); err != nil { + t.Error(err.Error()) + } + // Make sure we have removed it + if tms, err := mongoDb.GetTpTimings(utils.TEST_SQL, tm.TimingId); err != nil { + t.Error(err) + } else if len(tms) != 0 { + t.Errorf("Non empty timings: %+v", tms) + } + if rpfs, err := mongoDb.GetTpRatingProfiles(&mrp[0]); err != nil { + t.Error(err) + } else if len(rpfs) != 0 { + t.Errorf("Non empty rpfs: %+v", rpfs) + } + if aas, err := mongoDb.GetTpAccountActions(maa); err != nil { + t.Error(err) + } else if len(aas) != 0 { + t.Errorf("Non empty account actions: %+v", aas) + } +} + +func TestMongoSetCdr(t *testing.T) { + if !*testLocal { + return + } + cgrCdr1 := &CgrCdr{utils.TOR: utils.VOICE, utils.ACCID: "aaa1", utils.CDRHOST: "192.168.1.1", utils.REQTYPE: utils.META_RATED, utils.DIRECTION: "*out", utils.TENANT: "cgrates.org", + utils.CATEGORY: "call", utils.ACCOUNT: "1001", utils.SUBJECT: "1001", utils.DESTINATION: "1002", utils.SETUP_TIME: "2013-11-08T08:42:20Z", + utils.ANSWER_TIME: "2013-11-08T08:42:26Z", utils.USAGE: "10s", utils.PDD: "4s", utils.SUPPLIER: "SUPPL1", "field_extr1": "val_extr1", "fieldextr2": "valextr2", utils.CDRSOURCE: utils.TEST_SQL} + + cgrCdr2 := &CgrCdr{utils.TOR: utils.VOICE, utils.ACCID: "aaa2", utils.CDRHOST: "192.168.1.1", utils.REQTYPE: utils.META_PREPAID, utils.DIRECTION: "*out", utils.TENANT: "cgrates.org", + utils.CATEGORY: "call", utils.ACCOUNT: "1001", utils.SUBJECT: "1001", utils.DESTINATION: "1002", utils.SETUP_TIME: "2013-11-08T08:42:22Z", + utils.ANSWER_TIME: "2013-11-08T08:42:26Z", utils.USAGE: "20", utils.PDD: "7s", utils.SUPPLIER: "SUPPL1", "field_extr1": "val_extr1", "fieldextr2": "valextr2", "cdrsource": utils.TEST_SQL} + + cgrCdr3 := &CgrCdr{utils.TOR: utils.VOICE, utils.ACCID: "aaa3", utils.CDRHOST: "192.168.1.1", utils.REQTYPE: utils.META_RATED, utils.DIRECTION: "*out", utils.TENANT: "cgrates.org", + utils.CATEGORY: "premium_call", utils.ACCOUNT: "1002", utils.SUBJECT: "1002", utils.DESTINATION: "1001", utils.SETUP_TIME: "2013-11-07T08:42:24Z", + utils.ANSWER_TIME: "2013-11-07T08:42:26Z", utils.USAGE: "60s", utils.PDD: "4s", utils.SUPPLIER: "SUPPL1", "field_extr1": "val_extr1", "fieldextr2": "valextr2", "cdrsource": utils.TEST_SQL} + + cgrCdr4 := &CgrCdr{utils.TOR: utils.VOICE, utils.ACCID: "aaa4", utils.CDRHOST: "192.168.1.2", utils.REQTYPE: utils.META_PSEUDOPREPAID, utils.DIRECTION: "*out", utils.TENANT: "itsyscom.com", + utils.CATEGORY: "call", utils.ACCOUNT: "1001", utils.SUBJECT: "1001", utils.DESTINATION: "+4986517174964", utils.SETUP_TIME: "2013-11-07T08:42:21Z", + utils.ANSWER_TIME: "2013-11-07T08:42:26Z", utils.USAGE: "1m2s", utils.PDD: "4s", utils.SUPPLIER: "SUPPL1", "field_extr1": "val_extr1", "fieldextr2": "valextr2", "cdrsource": utils.TEST_SQL} + + cgrCdr5 := &CgrCdr{utils.TOR: utils.VOICE, utils.ACCID: "aaa5", utils.CDRHOST: "192.168.1.2", utils.REQTYPE: utils.META_POSTPAID, utils.DIRECTION: "*out", utils.TENANT: "itsyscom.com", + utils.CATEGORY: "call", utils.ACCOUNT: "1002", utils.SUBJECT: "1002", utils.DESTINATION: "+4986517174963", utils.SETUP_TIME: "2013-11-07T08:42:25Z", + utils.ANSWER_TIME: "2013-11-07T08:42:26Z", utils.USAGE: "15s", utils.PDD: "7s", utils.SUPPLIER: "SUPPL1", "field_extr1": "val_extr1", "fieldextr2": "valextr2", "cdrsource": utils.TEST_SQL} + + for _, cdr := range []*CgrCdr{cgrCdr1, cgrCdr2, cgrCdr3, cgrCdr4, cgrCdr5} { + if err := mongoDb.SetCdr(cdr.AsStoredCdr("")); err != nil { + t.Error(err.Error()) + } + } + strCdr1 := &StoredCdr{TOR: utils.VOICE, AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: utils.META_RATED, + Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", + SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), + Usage: time.Duration(10) * time.Second, Pdd: time.Duration(3) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201} + strCdr1.CgrId = utils.Sha1(strCdr1.AccId, strCdr1.SetupTime.String()) + strCdr2 := &StoredCdr{TOR: utils.VOICE, AccId: "bbb2", CdrHost: "192.168.1.2", CdrSource: "UNKNOWN2", ReqType: utils.META_PREPAID, + Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", + SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), + Usage: time.Duration(12) * time.Second, Pdd: time.Duration(4) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + MediationRunId: utils.DEFAULT_RUNID, Cost: 0.201} + strCdr2.CgrId = utils.Sha1(strCdr2.AccId, strCdr2.SetupTime.String()) + strCdr3 := &StoredCdr{TOR: utils.VOICE, AccId: "bbb3", CdrHost: "192.168.1.1", CdrSource: utils.TEST_SQL, ReqType: utils.META_RATED, + Direction: "*out", Tenant: "itsyscom.com", Category: "call", Account: "1002", Subject: "1000", Destination: "+4986517174963", + SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), + Usage: time.Duration(10) * time.Second, Pdd: time.Duration(2) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201} + strCdr3.CgrId = utils.Sha1(strCdr3.AccId, strCdr3.SetupTime.String()) + + for _, cdr := range []*StoredCdr{strCdr1, strCdr2, strCdr3} { + if err := mongoDb.SetCdr(cdr); err != nil { + t.Error(err.Error()) + } + } +} + +func TestMongoSetRatedCdr(t *testing.T) { + if !*testLocal { + return + } + strCdr1 := &StoredCdr{TOR: utils.VOICE, AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: utils.META_RATED, + Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", + SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), + Usage: time.Duration(10) * time.Second, Pdd: time.Duration(3) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201} + strCdr1.CgrId = utils.Sha1(strCdr1.AccId, strCdr1.SetupTime.String()) + strCdr2 := &StoredCdr{TOR: utils.VOICE, AccId: "bbb2", CdrHost: "192.168.1.2", CdrSource: "UNKNOWN", ReqType: utils.META_PREPAID, + Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", + SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), + Usage: time.Duration(12) * time.Second, Pdd: time.Duration(7) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + MediationRunId: utils.DEFAULT_RUNID, Cost: 0.201} + strCdr2.CgrId = utils.Sha1(strCdr2.AccId, strCdr2.SetupTime.String()) + strCdr3 := &StoredCdr{TOR: utils.VOICE, AccId: "bbb3", CdrHost: "192.168.1.1", CdrSource: utils.TEST_SQL, ReqType: utils.META_RATED, + Direction: "*out", Tenant: "itsyscom.com", Category: "call", Account: "1002", Subject: "1002", Destination: "+4986517174964", + SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), + Usage: time.Duration(10) * time.Second, Pdd: time.Duration(2) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + MediationRunId: "wholesale_run", Cost: 1.201} + strCdr3.CgrId = utils.Sha1(strCdr3.AccId, strCdr3.SetupTime.String()) + + for _, cdr := range []*StoredCdr{strCdr1, strCdr2, strCdr3} { + if err := mongoDb.SetRatedCdr(cdr); err != nil { + t.Error(err.Error()) + } + } +} + +func TestMongoCallCost(t *testing.T) { + if !*testLocal { + return + } + cgrId := utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()) + cc := &CallCost{ + Direction: "*out", + Category: "call", + Tenant: "cgrates.org", + Subject: "91001", + Account: "8001", + Destination: "1002", + TOR: utils.VOICE, + Timespans: []*TimeSpan{ + &TimeSpan{ + TimeStart: time.Date(2013, 9, 10, 13, 40, 0, 0, time.UTC), + TimeEnd: time.Date(2013, 9, 10, 13, 41, 0, 0, time.UTC), + }, + &TimeSpan{ + TimeStart: time.Date(2013, 9, 10, 13, 41, 0, 0, time.UTC), + TimeEnd: time.Date(2013, 9, 10, 13, 41, 30, 0, time.UTC), + }, + }, + } + if err := mongoDb.LogCallCost(cgrId, utils.TEST_SQL, utils.DEFAULT_RUNID, cc); err != nil { + t.Error(err.Error()) + } + if ccRcv, err := mongoDb.GetCallCostLog(cgrId, utils.TEST_SQL, utils.DEFAULT_RUNID); err != nil { + t.Error(err.Error()) + } else if !reflect.DeepEqual(cc.Timespans[0].TimeStart, ccRcv.Timespans[0].TimeStart) { + t.Errorf("Expecting call cost:\n%+v,\nreceived:\n%+v", cc.Timespans[0], ccRcv.Timespans[0]) + } + // UPDATE test here + cc.Category = "premium_call" + if err := mongoDb.LogCallCost(cgrId, utils.TEST_SQL, utils.DEFAULT_RUNID, cc); err != nil { + t.Error(err.Error()) + } + if ccRcv, err := mongoDb.GetCallCostLog(cgrId, utils.TEST_SQL, utils.DEFAULT_RUNID); err != nil { + t.Error(err.Error()) + } else if !reflect.DeepEqual(cc, ccRcv) { + t.Errorf("Expecting call cost: %v, received: %v", cc, ccRcv) + } +} + +func TestMongoGetStoredCdrs(t *testing.T) { + if !*testLocal { + return + } + var timeStart, timeEnd time.Time + // All CDRs, no filter + if storedCdrs, _, err := mongoDb.GetStoredCdrs(new(utils.CdrsFilter)); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 8 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Count ALL + if storedCdrs, count, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Count: true}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 0 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } else if count != 8 { + t.Error("Unexpected count of StoredCdrs returned: ", count) + } + // Limit 5 + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Paginator: utils.Paginator{Limit: utils.IntPointer(5), Offset: utils.IntPointer(0)}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 5 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Offset 5 + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Paginator: utils.Paginator{Limit: utils.IntPointer(5), Offset: utils.IntPointer(0)}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 5 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Offset with limit 2 + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Paginator: utils.Paginator{Limit: utils.IntPointer(2), Offset: utils.IntPointer(5)}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 2 { + t.Error("Unexpected number of StoredCdrs returned: ", len(storedCdrs)) + } + // Filter on cgrids + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{CgrIds: []string{utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()), + utils.Sha1("bbb2", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String())}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 2 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Count on CGRIDS + if _, count, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{CgrIds: []string{utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()), + utils.Sha1("bbb2", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String())}, Count: true}); err != nil { + t.Error(err.Error()) + } else if count != 2 { + t.Error("Unexpected count of StoredCdrs returned: ", count) + } + // Filter on cgrids plus reqType + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{CgrIds: []string{utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()), + utils.Sha1("bbb2", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String())}, ReqTypes: []string{utils.META_PREPAID}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 1 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Count on multiple filter + if _, count, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{CgrIds: []string{utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()), + utils.Sha1("bbb2", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String())}, ReqTypes: []string{utils.META_PREPAID}, Count: true}); err != nil { + t.Error(err.Error()) + } else if count != 1 { + t.Error("Unexpected count of StoredCdrs returned: ", count) + } + // Filter on runId + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{RunIds: []string{utils.DEFAULT_RUNID}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 2 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on TOR + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Tors: []string{utils.SMS}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 0 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on multiple TOR + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Tors: []string{utils.SMS, utils.VOICE}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 8 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on cdrHost + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{CdrHosts: []string{"192.168.1.2"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 3 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on multiple cdrHost + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{CdrHosts: []string{"192.168.1.1", "192.168.1.2"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 8 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on cdrSource + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{CdrSources: []string{"UNKNOWN"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 1 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on multiple cdrSource + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{CdrSources: []string{"UNKNOWN", "UNKNOWN2"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 2 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on reqType + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{ReqTypes: []string{utils.META_PREPAID}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 2 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on multiple reqType + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{ReqTypes: []string{utils.META_PREPAID, utils.META_PSEUDOPREPAID}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 3 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on direction + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Directions: []string{"*out"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 8 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on tenant + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Tenants: []string{"itsyscom.com"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 3 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on multiple tenants + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Tenants: []string{"itsyscom.com", "cgrates.org"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 8 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on category + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Categories: []string{"premium_call"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 1 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on multiple categories + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Categories: []string{"premium_call", "call"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 8 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on account + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Accounts: []string{"1002"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 3 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on multiple account + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Accounts: []string{"1001", "1002"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 8 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on subject + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Subjects: []string{"1000"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 1 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on multiple subject + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{Subjects: []string{"1000", "1002"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 3 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on destPrefix + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{DestPrefixes: []string{"+498651"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 3 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on multiple destPrefixes + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{DestPrefixes: []string{"1001", "+498651"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 4 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on ratedAccount + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{RatedAccounts: []string{"8001"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 1 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on ratedSubject + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{RatedSubjects: []string{"91001"}}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 1 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on ignoreRated + var orderIdStart, orderIdEnd int64 // Capture also orderIds for the next test + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{MaxCost: utils.Float64Pointer(0.0)}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 5 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } else { + for _, cdr := range storedCdrs { + if cdr.OrderId < orderIdStart { + orderIdStart = cdr.OrderId + } + if cdr.OrderId > orderIdEnd { + orderIdEnd = cdr.OrderId + } + } + } + // Filter on orderIdStart + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{OrderIdStart: orderIdStart}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 8 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on orderIdStart and orderIdEnd + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{OrderIdStart: orderIdStart, OrderIdEnd: orderIdEnd}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 4 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on timeStart + timeStart = time.Date(2013, 11, 8, 8, 0, 0, 0, time.UTC) + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{AnswerTimeStart: &timeStart}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 5 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on timeStart and timeEnd + timeEnd = time.Date(2013, 12, 1, 8, 0, 0, 0, time.UTC) + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{AnswerTimeStart: &timeStart, AnswerTimeEnd: &timeEnd}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 2 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on minPdd + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{MinPdd: utils.Float64Pointer(3)}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 7 { + t.Error("Unexpected number of StoredCdrs returned: ", len(storedCdrs)) + } + // Filter on maxPdd + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{MaxPdd: utils.Float64Pointer(3)}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 1 { + t.Error("Unexpected number of StoredCdrs returned: ", len(storedCdrs)) + } + // Filter on minPdd, maxPdd + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{MinPdd: utils.Float64Pointer(3), MaxPdd: utils.Float64Pointer(5)}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 5 { + t.Error("Unexpected number of StoredCdrs returned: ", len(storedCdrs)) + } + // Combined filter + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{ReqTypes: []string{utils.META_RATED}, AnswerTimeStart: &timeStart, AnswerTimeEnd: &timeEnd}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 1 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + // Filter on ignoreDerived + if storedCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{AnswerTimeStart: &timeStart, AnswerTimeEnd: &timeEnd, FilterOnRated: true}); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 0 { // ToDo: Recheck this value + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } +} + +func TestMongoRemStoredCdrs(t *testing.T) { + if !*testLocal { + return + } + cgrIdB1 := utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()) + if err := mongoDb.RemStoredCdrs([]string{cgrIdB1}); err != nil { + t.Error(err.Error()) + } + if storedCdrs, _, err := mongoDb.GetStoredCdrs(new(utils.CdrsFilter)); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 7 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } + tm, _ := utils.ParseTimeDetectLayout("2013-11-08T08:42:20Z", "") + cgrIdA1 := utils.Sha1("aaa1", tm.String()) + tm, _ = utils.ParseTimeDetectLayout("2013-11-08T08:42:22Z", "") + cgrIdA2 := utils.Sha1("aaa2", tm.String()) + tm, _ = utils.ParseTimeDetectLayout("2013-11-07T08:42:24Z", "") + cgrIdA3 := utils.Sha1("aaa3", tm.String()) + tm, _ = utils.ParseTimeDetectLayout("2013-11-07T08:42:21Z", "") + cgrIdA4 := utils.Sha1("aaa4", tm.String()) + tm, _ = utils.ParseTimeDetectLayout("2013-11-07T08:42:25Z", "") + cgrIdA5 := utils.Sha1("aaa5", tm.String()) + cgrIdB2 := utils.Sha1("bbb2", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()) + cgrIdB3 := utils.Sha1("bbb3", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()) + if err := mongoDb.RemStoredCdrs([]string{cgrIdA1, cgrIdA2, cgrIdA3, cgrIdA4, cgrIdA5, + cgrIdB2, cgrIdB3}); err != nil { + t.Error(err.Error()) + } + if storedCdrs, _, err := mongoDb.GetStoredCdrs(new(utils.CdrsFilter)); err != nil { + t.Error(err.Error()) + } else if len(storedCdrs) != 0 { + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + } +} + +// Make sure that what we get is what we set +func TestMongoStoreRestoreCdr(t *testing.T) { + if !*testLocal { + return + } + strCdr := &StoredCdr{TOR: utils.VOICE, AccId: "ccc1", CdrHost: "192.168.1.1", CdrSource: "TEST_CDR", ReqType: utils.META_RATED, + Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", + SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), + Usage: time.Duration(10) * time.Second, Pdd: time.Duration(3) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201} + strCdr.CgrId = utils.Sha1(strCdr.AccId, strCdr.SetupTime.String()) + if err := mongoDb.SetCdr(strCdr); err != nil { + t.Error(err.Error()) + } + if err := mongoDb.SetRatedCdr(strCdr); err != nil { + t.Error(err.Error()) + } + // Check RawCdr + if rcvCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{CgrIds: []string{strCdr.CgrId}}); err != nil { + t.Error(err.Error()) + } else if len(rcvCdrs) != 1 { + t.Errorf("Unexpected cdrs returned: %+v", rcvCdrs) + } else { + rcvCdr := rcvCdrs[0] + if strCdr.CgrId != rcvCdr.CgrId || + strCdr.TOR != rcvCdr.TOR || + strCdr.AccId != rcvCdr.AccId || + strCdr.CdrHost != rcvCdr.CdrHost || + strCdr.ReqType != rcvCdr.ReqType || + strCdr.Direction != rcvCdr.Direction || + strCdr.Tenant != rcvCdr.Tenant || + strCdr.Category != rcvCdr.Category || + strCdr.Account != rcvCdr.Account || + strCdr.Subject != rcvCdr.Subject || + strCdr.Destination != rcvCdr.Destination || + !strCdr.SetupTime.Equal(rcvCdr.SetupTime) || + !strCdr.AnswerTime.Equal(rcvCdr.AnswerTime) || + strCdr.Usage != rcvCdr.Usage || + strCdr.Pdd != rcvCdr.Pdd || + strCdr.Supplier != rcvCdr.Supplier || + strCdr.DisconnectCause != rcvCdr.DisconnectCause || + !reflect.DeepEqual(strCdr.ExtraFields, rcvCdr.ExtraFields) { + t.Errorf("Expecting: %+v, received: %+v", strCdr, rcvCdrs[0]) + } + } + // Check RatedCdr + if rcvCdrs, _, err := mongoDb.GetStoredCdrs(&utils.CdrsFilter{CgrIds: []string{strCdr.CgrId}, FilterOnRated: true}); err != nil { + t.Error(err.Error()) + } else if len(rcvCdrs) != 1 { + t.Errorf("Unexpected cdrs returned: %+v", rcvCdrs) + } else { + rcvCdr := rcvCdrs[0] + if strCdr.CgrId != rcvCdr.CgrId || + strCdr.TOR != rcvCdr.TOR || + strCdr.AccId != rcvCdr.AccId || + strCdr.CdrHost != rcvCdr.CdrHost || + strCdr.ReqType != rcvCdr.ReqType || + strCdr.Direction != rcvCdr.Direction || + strCdr.Tenant != rcvCdr.Tenant || + strCdr.Category != rcvCdr.Category || + strCdr.Account != rcvCdr.Account || + strCdr.Subject != rcvCdr.Subject || + strCdr.Destination != rcvCdr.Destination || + //!strCdr.SetupTime.Equal(rcvCdr.SetupTime) || // FixMe + //!strCdr.AnswerTime.Equal(rcvCdr.AnswerTime) || // FixMe + strCdr.Usage != rcvCdr.Usage || + strCdr.Pdd != rcvCdr.Pdd || + strCdr.Supplier != rcvCdr.Supplier || + strCdr.DisconnectCause != rcvCdr.DisconnectCause || + strCdr.Cost != rcvCdr.Cost || + !reflect.DeepEqual(strCdr.ExtraFields, rcvCdr.ExtraFields) { + t.Errorf("Expecting: %+v, received: %+v", strCdr, rcvCdrs[0]) + } + } +} diff --git a/engine/storage_mongo_tp.go b/engine/storage_mongo_tp.go index 77a5af084..921481e7a 100644 --- a/engine/storage_mongo_tp.go +++ b/engine/storage_mongo_tp.go @@ -360,13 +360,17 @@ func (ms *MongoStorage) RemTpData(table, tpid string, args map[string]string) er } for _, col := range cols { if strings.HasPrefix(col, "tp_") { - if err := ms.db.C(col).Remove(bson.M{"tpid": tpid}); err != nil { + if _, err := ms.db.C(col).RemoveAll(bson.M{"tpid": tpid}); err != nil { return err } } } + return nil } // Remove from a single table + if args == nil { + args = make(map[string]string) + } args["tpid"] = tpid return ms.db.C(table).Remove(args) } @@ -698,12 +702,14 @@ func (ms *MongoStorage) LogActionPlan(source string, at *ActionPlan, as Actions) } func (ms *MongoStorage) LogCallCost(cgrid, source, runid string, cc *CallCost) error { - return ms.db.C(colLogCC).Insert(&struct { + s := &struct { Id string `bson:"id,omitempty"` Source string Runid string `bson:"runid,omitempty"` CallCost *CallCost - }{cgrid, source, runid, cc}) + }{cgrid, source, runid, cc} + _, err := ms.db.C(colLogCC).Upsert(bson.M{"id": s.Id, "source": s.Source, "runid": s.Runid}, s) + return err } func (ms *MongoStorage) GetCallCostLog(cgrid, source, runid string) (cc *CallCost, err error) { @@ -713,17 +719,19 @@ func (ms *MongoStorage) GetCallCostLog(cgrid, source, runid string) (cc *CallCos Runid string `bson:"runid,omitempty"` CallCost *CallCost }{} - err = ms.db.C(colLogCC).Find(bson.M{"_id": cgrid, "source": source}).One(result) + err = ms.db.C(colLogCC).Find(bson.M{"id": cgrid, "source": source, "runid": runid}).One(result) cc = result.CallCost return } func (ms *MongoStorage) SetCdr(cdr *StoredCdr) error { - return ms.db.C(colCdrs).Insert(cdr) + _, err := ms.db.C(colCdrs).Upsert(bson.M{"cgrid": cdr.CgrId}, cdr) + return err } func (ms *MongoStorage) SetRatedCdr(storedCdr *StoredCdr) error { - return ms.db.C(colRatedCdrs).Insert(storedCdr) + _, err := ms.db.C(colRatedCdrs).Upsert(bson.M{"cgrid": storedCdr.CgrId}, storedCdr) + return err } // Remove CDR data out of all CDR tables based on their cgrid diff --git a/engine/storage_mysql.go b/engine/storage_mysql.go index 0ee327a0f..539a242e1 100644 --- a/engine/storage_mysql.go +++ b/engine/storage_mysql.go @@ -28,7 +28,7 @@ import ( "github.com/jinzhu/gorm" ) -func NewMySQLStorage(host, port, name, user, password string, maxConn, maxIdleConn int) (Storage, error) { +func NewMySQLStorage(host, port, name, user, password string, maxConn, maxIdleConn int) (*MySQLStorage, error) { connectString := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8&loc=Local&parseTime=true", user, password, host, port, name) db, err := gorm.Open("mysql", connectString) if err != nil { diff --git a/engine/storage_mysql_local_test.go b/engine/storage_mysql_local_test.go index 5f8a974e5..6000d307d 100644 --- a/engine/storage_mysql_local_test.go +++ b/engine/storage_mysql_local_test.go @@ -36,12 +36,11 @@ func TestMySQLCreateTables(t *testing.T) { return } cgrConfig, _ := config.NewDefaultCGRConfig() - if d, err := NewMySQLStorage(cgrConfig.StorDBHost, cgrConfig.StorDBPort, cgrConfig.StorDBName, cgrConfig.StorDBUser, cgrConfig.StorDBPass, + var err error + if mysqlDb, err = NewMySQLStorage(cgrConfig.StorDBHost, cgrConfig.StorDBPort, cgrConfig.StorDBName, cgrConfig.StorDBUser, cgrConfig.StorDBPass, cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns); err != nil { t.Error("Error on opening database connection: ", err) return - } else { - mysqlDb = d.(*MySQLStorage) } for _, scriptName := range []string{utils.CREATE_CDRS_TABLES_SQL, utils.CREATE_TARIFFPLAN_TABLES_SQL} { if err := mysqlDb.CreateTablesFromScript(path.Join(*dataDir, "storage", utils.MYSQL, scriptName)); err != nil { diff --git a/engine/storage_postgres.go b/engine/storage_postgres.go index dda045f96..8641e828b 100644 --- a/engine/storage_postgres.go +++ b/engine/storage_postgres.go @@ -34,7 +34,7 @@ type PostgresStorage struct { *SQLStorage } -func NewPostgresStorage(host, port, name, user, password string, maxConn, maxIdleConn int) (Storage, error) { +func NewPostgresStorage(host, port, name, user, password string, maxConn, maxIdleConn int) (*PostgresStorage, error) { connectString := fmt.Sprintf("host=%s port=%s dbname=%s user=%s password=%s sslmode=disable", host, port, name, user, password) db, err := gorm.Open("postgres", connectString) if err != nil { diff --git a/engine/storage_psql_local_test.go b/engine/storage_psql_local_test.go index 0388111ed..765639dee 100644 --- a/engine/storage_psql_local_test.go +++ b/engine/storage_psql_local_test.go @@ -36,12 +36,11 @@ func TestPSQLCreateTables(t *testing.T) { return } cgrConfig, _ := config.NewDefaultCGRConfig() - if d, err := NewPostgresStorage("localhost", "5432", cgrConfig.StorDBName, cgrConfig.StorDBUser, cgrConfig.StorDBPass, + var err error + if psqlDb, err = NewPostgresStorage("localhost", "5432", cgrConfig.StorDBName, cgrConfig.StorDBUser, cgrConfig.StorDBPass, cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns); err != nil { t.Error("Error on opening database connection: ", err) return - } else { - psqlDb = d.(*PostgresStorage) } for _, scriptName := range []string{utils.CREATE_CDRS_TABLES_SQL, utils.CREATE_TARIFFPLAN_TABLES_SQL} { if err := psqlDb.CreateTablesFromScript(path.Join(*dataDir, "storage", utils.POSTGRES, scriptName)); err != nil { diff --git a/engine/storage_utils.go b/engine/storage_utils.go index dc19abe43..ee2293d7a 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -51,7 +51,7 @@ func ConfigureRatingStorage(db_type, host, port, name, user, pass, marshaler str } func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler string) (db AccountingStorage, err error) { - var d Storage + var d AccountingStorage switch db_type { case utils.REDIS: var db_nb int @@ -73,11 +73,11 @@ func ConfigureAccountingStorage(db_type, host, port, name, user, pass, marshaler if err != nil { return nil, err } - return d.(AccountingStorage), nil + return d, nil } func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn int) (db LogStorage, err error) { - var d Storage + var d LogStorage switch db_type { /* case utils.REDIS: @@ -104,11 +104,11 @@ func ConfigureLogStorage(db_type, host, port, name, user, pass, marshaler string if err != nil { return nil, err } - return d.(LogStorage), nil + return d, nil } func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler string, maxConn, maxIdleConn int) (db LoadStorage, err error) { - var d Storage + var d LoadStorage switch db_type { case utils.POSTGRES: d, err = NewPostgresStorage(host, port, name, user, pass, maxConn, maxIdleConn) @@ -122,11 +122,11 @@ func ConfigureLoadStorage(db_type, host, port, name, user, pass, marshaler strin if err != nil { return nil, err } - return d.(LoadStorage), nil + return d, nil } func ConfigureCdrStorage(db_type, host, port, name, user, pass string, maxConn, maxIdleConn int) (db CdrStorage, err error) { - var d Storage + var d CdrStorage switch db_type { case utils.POSTGRES: d, err = NewPostgresStorage(host, port, name, user, pass, maxConn, maxIdleConn) @@ -140,5 +140,5 @@ func ConfigureCdrStorage(db_type, host, port, name, user, pass string, maxConn, if err != nil { return nil, err } - return d.(CdrStorage), nil + return d, nil } diff --git a/general_tests/multiplecdrc_local_test.go b/general_tests/multiplecdrc_local_test.go index 55624b294..f651e2a7c 100644 --- a/general_tests/multiplecdrc_local_test.go +++ b/general_tests/multiplecdrc_local_test.go @@ -84,11 +84,9 @@ func TestEmptyTables(t *testing.T) { if *storDbType != utils.MYSQL { t.Fatal("Unsupported storDbType") } - var mysql *engine.MySQLStorage - if d, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns); err != nil { + mysql, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns) + if err != nil { t.Fatal("Error on opening database connection: ", err) - } else { - mysql = d.(*engine.MySQLStorage) } if err := mysql.CreateTablesFromScript(path.Join(*dataDir, "storage", *storDbType, utils.CREATE_CDRS_TABLES_SQL)); err != nil { t.Fatal("Error on mysql creation: ", err.Error()) From a744a6f5926db08822131b7b99a2636138fdfd3f Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 13 Oct 2015 16:42:07 +0300 Subject: [PATCH 7/7] added apier cdrs mongo local tests --- apier/v2/cdrs_mongo_local_test.go | 243 ++++++++++++++++++ data/conf/samples/cdrsv2mongo/cdrsv2psql.json | 22 ++ 2 files changed, 265 insertions(+) create mode 100644 apier/v2/cdrs_mongo_local_test.go create mode 100644 data/conf/samples/cdrsv2mongo/cdrsv2psql.json diff --git a/apier/v2/cdrs_mongo_local_test.go b/apier/v2/cdrs_mongo_local_test.go new file mode 100644 index 000000000..5bca5fd15 --- /dev/null +++ b/apier/v2/cdrs_mongo_local_test.go @@ -0,0 +1,243 @@ +/* +Real-time Charging System for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can Storagetribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITH*out ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package v2 + +import ( + "net/rpc" + "net/rpc/jsonrpc" + "path" + "testing" + "time" + + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/engine" + "github.com/cgrates/cgrates/utils" +) + +var cdrsMongoCfgPath string +var cdrsMongoCfg *config.CGRConfig +var cdrsMongoRpc *rpc.Client + +func TestV2CdrsMongoInitConfig(t *testing.T) { + if !*testLocal { + return + } + var err error + cdrsMongoCfgPath = path.Join(*dataDir, "conf", "samples", "cdrsv2mongo") + if cdrsMongoCfg, err = config.NewCGRConfigFromFolder(cdrsMongoCfgPath); err != nil { + t.Fatal("Got config error: ", err.Error()) + } +} + +func TestV2CdrsMongoInitDataDb(t *testing.T) { + if !*testLocal { + return + } + if err := engine.InitDataDb(cdrsMongoCfg); err != nil { + t.Fatal(err) + } +} + +// InitDb so we can rely on count +func TestV2CdrsMongoInitCdrDb(t *testing.T) { + if !*testLocal { + return + } + if err := engine.InitStorDb(cdrsMongoCfg); err != nil { + t.Fatal(err) + } +} + +func TestV2CdrsMongoInjectUnratedCdr(t *testing.T) { + if !*testLocal { + return + } + mysqlDb, err := engine.NewMongoStorage(cdrsMongoCfg.StorDBHost, cdrsMongoCfg.StorDBPort, cdrsMongoCfg.StorDBName, cdrsMongoCfg.StorDBUser, cdrsMongoCfg.StorDBPass) + if err != nil { + t.Error("Error on opening database connection: ", err) + return + } + strCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()), + TOR: utils.VOICE, AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: utils.META_RATED, + Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", + SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), + Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201} + if err := mysqlDb.SetCdr(strCdr1); err != nil { + t.Error(err.Error()) + } +} + +func TestV2CdrsMongoStartEngine(t *testing.T) { + if !*testLocal { + return + } + if _, err := engine.StopStartEngine(cdrsMongoCfgPath, *waitRater); err != nil { + t.Fatal(err) + } +} + +// Connect rpc client to rater +func TestV2CdrsMongoRpcConn(t *testing.T) { + if !*testLocal { + return + } + var err error + cdrsMongoRpc, err = jsonrpc.Dial("tcp", cdrsMongoCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed + if err != nil { + t.Fatal("Could not connect to rater: ", err.Error()) + } +} + +// Insert some CDRs +func TestV2CdrsMongoProcessCdr(t *testing.T) { + if !*testLocal { + return + } + var reply string + cdrs := []*engine.StoredCdr{ + &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", + CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", + SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, + Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + RatedAccount: "dan", RatedSubject: "dans", Rated: true, + }, + &engine.StoredCdr{CgrId: utils.Sha1("abcdeftg", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", + CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002", + SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, + Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + RatedAccount: "dan", RatedSubject: "dans", + }, + &engine.StoredCdr{CgrId: utils.Sha1("aererfddf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", + CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002", + SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, + Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + RatedAccount: "dan", RatedSubject: "dans", + }, + } + for _, cdr := range cdrs { + if err := cdrsMongoRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if reply != utils.OK { + t.Error("Unexpected reply received: ", reply) + } + } +} + +func TestV2CdrsMongoGetCdrs(t *testing.T) { + if !*testLocal { + return + } + var reply []*engine.ExternalCdr + req := utils.RpcCdrsFilter{} + if err := cdrsMongoRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(reply) != 4 { + t.Error("Unexpected number of CDRs returned: ", len(reply)) + } + // CDRs with errors + req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0), MaxCost: utils.Float64Pointer(0.0)} + if err := cdrsMongoRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(reply) != 2 { + t.Error("Unexpected number of CDRs returned: ", reply) + } + // CDRs Rated + req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0)} + if err := cdrsMongoRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(reply) != 3 { + t.Error("Unexpected number of CDRs returned: ", reply) + } + // CDRs non rated OR SkipRated + req = utils.RpcCdrsFilter{MaxCost: utils.Float64Pointer(-1.0)} + if err := cdrsMongoRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(reply) != 1 { + t.Error("Unexpected number of CDRs returned: ", reply) + } + // Skip Errors + req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(0.0), MaxCost: utils.Float64Pointer(-1.0)} + if err := cdrsMongoRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if len(reply) != 2 { + t.Error("Unexpected number of CDRs returned: ", reply) + } +} + +func TestV2CdrsMongoCountCdrs(t *testing.T) { + if !*testLocal { + return + } + var reply int64 + req := utils.AttrGetCdrs{} + if err := cdrsMongoRpc.Call("ApierV2.CountCdrs", req, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if reply != 4 { + t.Error("Unexpected number of CDRs returned: ", reply) + } +} + +// Test Prepaid CDRs without previous costs being calculated +func TestV2CdrsMongoProcessPrepaidCdr(t *testing.T) { + if !*testLocal { + return + } + var reply string + cdrs := []*engine.StoredCdr{ + &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", + CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", + SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, + Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + RatedAccount: "dan", RatedSubject: "dans", Rated: true, + }, + &engine.StoredCdr{CgrId: utils.Sha1("abcdeftg2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", + CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002", + SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, + Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + RatedAccount: "dan", RatedSubject: "dans", + }, + &engine.StoredCdr{CgrId: utils.Sha1("aererfddf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", + CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002", + SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, + Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + RatedAccount: "dan", RatedSubject: "dans", + }, + } + tStart := time.Now() + for _, cdr := range cdrs { + if err := cdrsMongoRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if reply != utils.OK { + t.Error("Unexpected reply received: ", reply) + } + } + if processDur := time.Now().Sub(tStart); processDur > 1*time.Second { + t.Error("Unexpected processing time", processDur) + } +} + +func TestV2CdrsMongoKillEngine(t *testing.T) { + if !*testLocal { + return + } + if err := engine.KillEngine(*waitRater); err != nil { + t.Error(err) + } +} diff --git a/data/conf/samples/cdrsv2mongo/cdrsv2psql.json b/data/conf/samples/cdrsv2mongo/cdrsv2psql.json new file mode 100644 index 000000000..e62a46f95 --- /dev/null +++ b/data/conf/samples/cdrsv2mongo/cdrsv2psql.json @@ -0,0 +1,22 @@ +{ +// CGRateS Configuration file +// +// Used in apier_local_tests +// Starts rater, cdrs and mediator connecting over internal channel + +"stor_db": { + "db_type": "mongo", // stor database type to use: + "db_port": 27017, // the port to reach the stordb +}, + + +"rater": { + "enabled": true, // enable Rater service: +}, + +"cdrs": { + "enabled": true, // start the CDR Server service: + "rater": "internal", // address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234> +}, + +} \ No newline at end of file