From c9211a928a28eaaf2e5055be89a8967fd935b1cc Mon Sep 17 00:00:00 2001 From: Tripon Alexandru-Ionut Date: Fri, 19 Apr 2019 17:33:05 +0300 Subject: [PATCH 1/5] Updated EnsureIndexes --- engine/storage_mongo_datadb.go | 121 ++++++++++++++++++++------------- 1 file changed, 74 insertions(+), 47 deletions(-) diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 77cf247a8..20292d7ab 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -226,7 +226,7 @@ func (ms *MongoStorage) IsDataDB() bool { return ms.isDataDB } -func (ms *MongoStorage) EnusureIndex(colName string, uniq bool, keys ...string) error { +func (ms *MongoStorage) enusureIndex(colName string, uniq bool, keys ...string) error { return ms.query(func(sctx mongo.SessionContext) error { col := ms.getCol(colName) io := options.Index().SetUnique(uniq) @@ -242,6 +242,14 @@ func (ms *MongoStorage) EnusureIndex(colName string, uniq bool, keys ...string) }) } +func (ms *MongoStorage) dropAllIndexesForCol(colName string) error { + return ms.query(func(sctx mongo.SessionContext) error { + col := ms.getCol(colName) + _, err := col.Indexes().DropAll(sctx) + return err + }) +} + func (ms *MongoStorage) getCol(col string) *mongo.Collection { return ms.client.Database(ms.db).Collection(col) } @@ -250,23 +258,72 @@ func (ms *MongoStorage) GetContext() context.Context { return ms.ctx } +func (ms *MongoStorage) EnsureIndexesForCol(col string) (err error) { // exported for migrator + if err = ms.dropAllIndexesForCol(col); err != nil { // make sure you do not have indexes + return + } + switch col { + case colAct, colApl, colAAp, colAtr, colRpl, colDst, colRds, colLht, colRFI: + if err = ms.enusureIndex(col, true, "key"); err != nil { + return + } + case colRsP, colRes, colSqs, colSqp, colTps, colThs, colSpp, colAttr, colFlt, colCpp, colDpp: + if err = ms.enusureIndex(col, true, "tenant", "id"); err != nil { + return + } + case colRpf, colShg, colAcc: + if err = ms.enusureIndex(col, true, "id"); err != nil { + return + } + //StorDB + case utils.TBLTPTimings, utils.TBLTPDestinations, + utils.TBLTPDestinationRates, utils.TBLTPRatingPlans, + utils.TBLTPSharedGroups, utils.TBLTPActions, + utils.TBLTPActionPlans, utils.TBLTPActionTriggers, + utils.TBLTPStats, utils.TBLTPResources: + if err = ms.enusureIndex(col, true, "tpid", "id"); err != nil { + return + } + case utils.TBLTPRateProfiles: + if err = ms.enusureIndex(col, true, "tpid", "direction", + "tenant", "category", "subject", "loadid"); err != nil { + return + } + case utils.CDRsTBL: + if err = ms.enusureIndex(col, true, CGRIDLow, RunIDLow, + OriginIDLow); err != nil { + return + } + for _, idxKey := range ms.cdrsIndexes { + if err = ms.enusureIndex(col, false, idxKey); err != nil { + return + } + } + case utils.SessionCostsTBL: + if err = ms.enusureIndex(col, true, CGRIDLow, + RunIDLow); err != nil { + return + } + if err = ms.enusureIndex(col, false, OriginHostLow, + OriginIDLow); err != nil { + return + } + if err = ms.enusureIndex(col, false, RunIDLow, + OriginIDLow); err != nil { + return + } + } + return +} + // EnsureIndexes creates db indexes func (ms *MongoStorage) EnsureIndexes() (err error) { if ms.storageType == utils.DataDB { for _, col := range []string{colAct, colApl, colAAp, colAtr, - colRpl, colDst, colRds, colLht, colRFI} { - if err = ms.EnusureIndex(col, true, "key"); err != nil { - return - } - } - for _, col := range []string{colRsP, colRes, colSqs, colSqp, - colTps, colThs, colSpp, colAttr, colFlt, colCpp, colDpp} { - if err = ms.EnusureIndex(col, true, "tenant", "id"); err != nil { - return - } - } - for _, col := range []string{colRpf, colShg, colAcc} { - if err = ms.EnusureIndex(col, true, "id"); err != nil { + colRpl, colDst, colRds, colLht, colRFI, colRsP, colRes, colSqs, colSqp, + colTps, colThs, colSpp, colAttr, colFlt, colCpp, colDpp, + colRpf, colShg, colAcc} { + if err = ms.EnsureIndexesForCol(col); err != nil { return } } @@ -276,42 +333,12 @@ func (ms *MongoStorage) EnsureIndexes() (err error) { utils.TBLTPDestinationRates, utils.TBLTPRatingPlans, utils.TBLTPSharedGroups, utils.TBLTPActions, utils.TBLTPActionPlans, utils.TBLTPActionTriggers, - utils.TBLTPStats, utils.TBLTPResources} { - if err = ms.EnusureIndex(col, true, "tpid", "id"); err != nil { + utils.TBLTPStats, utils.TBLTPResources, + utils.TBLTPRateProfiles, utils.CDRsTBL, utils.SessionCostsTBL} { + if err = ms.EnsureIndexesForCol(col); err != nil { return } } - - if err = ms.EnusureIndex(utils.TBLTPRateProfiles, true, "tpid", "direction", - "tenant", "category", "subject", "loadid"); err != nil { - return - } - - if err = ms.EnusureIndex(utils.CDRsTBL, true, CGRIDLow, RunIDLow, - OriginIDLow); err != nil { - return - } - - for _, idxKey := range ms.cdrsIndexes { - if err = ms.EnusureIndex(utils.CDRsTBL, false, idxKey); err != nil { - return - } - } - - if err = ms.EnusureIndex(utils.SessionCostsTBL, true, CGRIDLow, - RunIDLow); err != nil { - return - } - - if err = ms.EnusureIndex(utils.SessionCostsTBL, false, OriginHostLow, - OriginIDLow); err != nil { - return - } - - if err = ms.EnusureIndex(utils.SessionCostsTBL, false, RunIDLow, - OriginIDLow); err != nil { - return - } } return } From b035267c89edd4c95bec14e757731d61d929c308 Mon Sep 17 00:00:00 2001 From: Tripon Alexandru-Ionut Date: Fri, 19 Apr 2019 17:40:23 +0300 Subject: [PATCH 2/5] Added *ensure_indexes option to migrator --- migrator/migrator.go | 18 ++++++++++++++++++ utils/consts.go | 1 + 2 files changed, 19 insertions(+) diff --git a/migrator/migrator.go b/migrator/migrator.go index 09ab8a14e..7cf276f68 100755 --- a/migrator/migrator.go +++ b/migrator/migrator.go @@ -87,6 +87,24 @@ func (m *Migrator) Migrate(taskIDs []string) (err error, stats map[string]int) { return utils.NewCGRError(utils.Migrator, utils.ServerErrorCaps, err.Error(), fmt.Sprintf("error: <%s> when seting versions for StorDB", err.Error())), nil } + case utils.MetaEnsureIndexes: + if m.storDBOut.StorDB().GetStorageType() == utils.MONGO { + mgo := m.storDBOut.StorDB().(*engine.MongoStorage) + if err = mgo.EnsureIndexes(); err != nil { + return + } + } else { + fmt.Printf("The StorDB type is not %s .\n %s works only on %s .\n", utils.MONGO, utils.MetaEnsureIndexes, utils.MONGO) + } + + if m.dmOut.DataManager().DataDB().GetStorageType() == utils.MONGO { + mgo := m.dmOut.DataManager().DataDB().(*engine.MongoStorage) + if err = mgo.EnsureIndexes(); err != nil { + return + } + } else { + fmt.Printf("The DataDB type is not %s .\n %s works only on %s.\n ", utils.MONGO, utils.MetaEnsureIndexes, utils.MONGO) + } case utils.MetaCDRs: err = m.migrateCDRs() case utils.MetaSessionsCosts: diff --git a/utils/consts.go b/utils/consts.go index 14cfe2351..1250567cd 100755 --- a/utils/consts.go +++ b/utils/consts.go @@ -627,6 +627,7 @@ const ( // Migrator Metas const ( MetaSetVersions = "*set_versions" + MetaEnsureIndexes = "*ensure_indexes" MetaTpRatingPlans = "*tp_rating_plans" MetaTpFilters = "*tp_filters" MetaTpDestinationRates = "*tp_destination_rates" From 5c0807393696beb51e58f2e87c29aee7c210cb7b Mon Sep 17 00:00:00 2001 From: Tripon Alexandru-Ionut Date: Fri, 19 Apr 2019 17:52:28 +0300 Subject: [PATCH 3/5] Updated *ensure_indexes message --- migrator/migrator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migrator/migrator.go b/migrator/migrator.go index 7cf276f68..30f670ac9 100755 --- a/migrator/migrator.go +++ b/migrator/migrator.go @@ -94,7 +94,7 @@ func (m *Migrator) Migrate(taskIDs []string) (err error, stats map[string]int) { return } } else { - fmt.Printf("The StorDB type is not %s .\n %s works only on %s .\n", utils.MONGO, utils.MetaEnsureIndexes, utils.MONGO) + log.Printf("The StorDB type has to be %s .\n ", utils.MONGO) } if m.dmOut.DataManager().DataDB().GetStorageType() == utils.MONGO { @@ -103,7 +103,7 @@ func (m *Migrator) Migrate(taskIDs []string) (err error, stats map[string]int) { return } } else { - fmt.Printf("The DataDB type is not %s .\n %s works only on %s.\n ", utils.MONGO, utils.MetaEnsureIndexes, utils.MONGO) + log.Printf("The DataDB type has to be %s .\n ", utils.MONGO) } case utils.MetaCDRs: err = m.migrateCDRs() From 618dc9d3ccb18e65eb7a9b8f20e932b30a1aea91 Mon Sep 17 00:00:00 2001 From: Tripon Alexandru-Ionut Date: Fri, 19 Apr 2019 18:16:08 +0300 Subject: [PATCH 4/5] Updated EnsureIndexesForCol --- engine/storage_mongo_datadb.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 20292d7ab..2bd14752f 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -41,6 +41,7 @@ import ( "github.com/mongodb/mongo-go-driver/mongo" "github.com/mongodb/mongo-go-driver/mongo/options" "github.com/mongodb/mongo-go-driver/x/bsonx" + "github.com/mongodb/mongo-go-driver/x/network/command" ) const ( @@ -259,7 +260,7 @@ func (ms *MongoStorage) GetContext() context.Context { } func (ms *MongoStorage) EnsureIndexesForCol(col string) (err error) { // exported for migrator - if err = ms.dropAllIndexesForCol(col); err != nil { // make sure you do not have indexes + if err = ms.dropAllIndexesForCol(col); err != nil && !command.IsNotFound(err) { // make sure you do not have indexes return } switch col { From 3eb113725c0174b4a3353589374f52550df52c0a Mon Sep 17 00:00:00 2001 From: Tripon Alexandru-Ionut Date: Fri, 19 Apr 2019 18:41:01 +0300 Subject: [PATCH 5/5] Updated matchingSupplierProfilesForEvent sorting function --- engine/suppliers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/suppliers.go b/engine/suppliers.go index f1347d418..f1bd63c05 100644 --- a/engine/suppliers.go +++ b/engine/suppliers.go @@ -168,7 +168,7 @@ func (spS *SupplierService) matchingSupplierProfilesForEvent(ev *utils.CGREvent, if len(matchingSLP) == 0 { return nil, utils.ErrNotFound } - sort.Slice(matchingSLP, func(i, j int) bool { return matchingSLP[i].Weight < matchingSLP[j].Weight }) + sort.Slice(matchingSLP, func(i, j int) bool { return matchingSLP[i].Weight > matchingSLP[j].Weight }) } return }