Added mongo new driver for datadb

engine/storage_mongo_datadb.go

@@ -1953,7 +1953,9 @@ func (ms *MongoStorage) SetFilterIndexesDrv(cacheID, itemIDPrefix string,
 			bulk := col.Bulk()
 			bulk.Unordered()
 			bulk.Upsert(pairs...)
-			_, err = bulk.Run()
+			if _, err = bulk.Run(); err != nil {
+				return err
+			}
 		}
 		oldKey := "tmp_" + utils.ConcatenatedKey(originKey, transactionID)
 		for _, character := range []string{".", "*"} {
@@ -1976,7 +1978,9 @@ func (ms *MongoStorage) SetFilterIndexesDrv(cacheID, itemIDPrefix string,
 			bulk := col.Bulk()
 			bulk.Unordered()
 			bulk.Upsert(pairs...)
-			_, err = bulk.Run()
+			if _, err = bulk.Run(); err != nil {
+				return err
+			}
 		}
 	}
 	return

engine/storage_mongo_stordb.go

@@ -80,7 +80,8 @@ func (ms *MongoStorage) GetTpTableIds(tpid, table string, distinct utils.TPDisti
 				Pattern: ".*" + regexp.QuoteMeta(pag.SearchTerm) + ".*",
 				Options: ""}})
 		}
-		findMap["$and"] = []bson.M{bson.M{"$or": searchItems}}
+		// findMap["$and"] = []bson.M{{"$or": searchItems}} //before
+		findMap["$or"] = searchItems // after
 	}
 
 	session, col := ms.conn(table)
@@ -1005,7 +1006,7 @@ func (ms *MongoStorage) GetCDRs(qryFltr *utils.CDRsFilter, remove bool) ([]*CDR,
 			filters[CostLow] = bson.M{"$gte": *qryFltr.MinCost}
 		} else if *qryFltr.MinCost == 0.0 && *qryFltr.MaxCost == -1.0 { // Special case when we want to skip errors
 			filters["$or"] = []bson.M{
-				bson.M{CostLow: bson.M{"$gte": 0.0}},
+				{CostLow: bson.M{"$gte": 0.0}},
 			}
 		} else {
 			filters[CostLow] = bson.M{"$gte": *qryFltr.MinCost, "$lt": *qryFltr.MaxCost}

engine/storage_mongo_stordb_new.go

@@ -0,0 +1,158 @@
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>
+*/
+
+package engine
+
+import (
+	// "fmt"
+	"context"
+	"regexp"
+	"strings"
+	// "time"
+
+	"github.com/cgrates/cgrates/utils"
+
+	"github.com/mongodb/mongo-go-driver/bson"
+	// "github.com/mongodb/mongo-go-driver/bson/objectid"
+	"github.com/mongodb/mongo-go-driver/mongo"
+	"github.com/mongodb/mongo-go-driver/options"
+	"github.com/mongodb/mongo-go-driver/x/bsonx"
+)
+
+func (ms *MongoStorageNew) GetTpIds(colName string) (tpids []string, err error) {
+	getTpIDs := func(ctx context.Context, col string, tpMap map[string]struct{}) (map[string]struct{}, error) {
+		if strings.HasPrefix(col, "tp_") {
+			result, err := ms.getCol(col).Distinct(ctx, "tpid", nil)
+			if err != nil {
+				return tpMap, err
+			}
+			for _, tpid := range result {
+				tpMap[tpid.(string)] = struct{}{}
+			}
+		}
+		return tpMap, nil
+	}
+	tpidMap := make(map[string]struct{})
+
+	if colName == "" {
+		if err := ms.client.UseSession(ms.ctx, func(sctx mongo.SessionContext) error {
+			col, err := ms.DB().ListCollections(sctx, nil, options.ListCollections().SetNameOnly(true))
+			if err != nil {
+				return err
+			}
+			for col.Next(sctx) {
+				var elem struct{ Name string }
+				if err := col.Decode(&elem); err != nil {
+					return err
+				}
+				if tpidMap, err = getTpIDs(sctx, elem.Name, tpidMap); err != nil {
+					return err
+				}
+			}
+			col.Close(sctx)
+			return nil
+		}); err != nil {
+			return nil, err
+		}
+	} else {
+		if err := ms.client.UseSession(ms.ctx, func(sctx mongo.SessionContext) error {
+			tpidMap, err = getTpIDs(sctx, colName, tpidMap)
+			return err
+		}); err != nil {
+			return nil, err
+		}
+	}
+	for tpid := range tpidMap {
+		tpids = append(tpids, tpid)
+	}
+	return tpids, nil
+}
+
+func (ms *MongoStorageNew) GetTpTableIds(tpid, table string, distinct utils.TPDistinctIds, filter map[string]string, pag *utils.Paginator) ([]string, error) {
+	findMap := bson.M{}
+	if tpid != "" {
+		findMap["tpid"] = tpid
+	}
+	for k, v := range filter {
+		findMap[k] = v
+	}
+	for k, v := range distinct { //fix for MongoStorage on TPUsers
+		if v == "user_name" {
+			distinct[k] = "username"
+		}
+	}
+	if pag != nil && pag.SearchTerm != "" {
+		var searchItems []bson.M
+		for _, d := range distinct {
+			searchItems = append(searchItems, bson.M{d: bsonx.Regex(".*"+regexp.QuoteMeta(pag.SearchTerm)+".*", "")})
+		}
+		// findMap["$and"] = []bson.M{{"$or": searchItems}} //before
+		findMap["$or"] = searchItems // after
+	}
+
+	fop := options.Find()
+	if pag != nil {
+		if pag.Limit != nil {
+			fop = fop.SetLimit(int64(*pag.Limit))
+		}
+		if pag.Offset != nil {
+			fop = fop.SetSkip(int64(*pag.Offset))
+		}
+	}
+
+	selectors := bson.M{"_id": 0}
+	for i, d := range distinct {
+		if d == "tag" { // convert the tag used in SQL into id used here
+			distinct[i] = "id"
+		}
+		selectors[distinct[i]] = 1
+	}
+	fop.SetProjection(selectors)
+
+	distinctIds := make(utils.StringMap)
+	if err := ms.client.UseSession(ms.ctx, func(sctx mongo.SessionContext) (err error) {
+		cur, err := ms.getCol(table).Find(sctx, findMap, fop)
+		if err != nil {
+			return err
+		}
+		for cur.Next(sctx) {
+			var elem bson.D
+			err := cur.Decode(&elem)
+			if err != nil {
+				return err
+			}
+			item := elem.Map()
+
+			var id string
+			last := len(distinct) - 1
+			for i, d := range distinct {
+				if distinctValue, ok := item[d]; ok {
+					id += distinctValue.(string)
+				}
+				if i < last {
+					id += utils.CONCATENATED_KEY_SEP
+				}
+			}
+			distinctIds[id] = true
+		}
+		return cur.Close(sctx)
+	}); err != nil {
+		return nil, err
+	}
+	return distinctIds.Slice(), nil
+}

glide.lock

@@ -1,6 +1,10 @@
-hash: 2daed4c5ed5052824ae5df18b0690a85226f9e3c61c336f83db7fccc6f4ecd51
-updated: 2017-04-21T14:51:07.386488974+02:00
+hash: 47448d94325e62dd3dcce88ebc5ad633efd329dea16f3f3a6f481cd4947eca89
+updated: 2018-11-19T14:47:52.199551832+02:00
 imports:
+- name: github.com/antchfx/xmlquery
+  version: 98cdbc3221ed492316cca5a4711e48ee0ca0dc9a
+- name: github.com/antchfx/xpath
+  version: a762252498f5e50d8f368dd0a8748073e4b92f93
 - name: github.com/cenkalti/hub
   version: 11382a9960d39b0ecda16fd01c424c11ff765a34
 - name: github.com/cenkalti/rpc2
@@ -15,14 +19,23 @@ imports:
   version: 0e0d0379606fd8f12b53c6da6aeb28544f7bfa37
 - name: github.com/cgrates/ltcache
   version: 92fb7fa77cca400b55d805e4a6d625443027c7f5
+- name: github.com/cgrates/mgo
+  version: 04b6cd516663d1f91d8f914e66ae743d3ab1685c
+  subpackages:
+  - bson
+  - internal/json
+  - internal/sasl
+  - internal/scram
 - name: github.com/cgrates/osipsdagram
   version: 3d6beed663452471dec3ca194137a30d379d9e8f
 - name: github.com/cgrates/radigo
   version: e5c8f3272cccf795f47b82e25a8f3408312c14e0
 - name: github.com/cgrates/rpcclient
   version: 7316bff37a2b8692fbadd57f9c9cda070cc33081
+- name: github.com/dlintw/goconf
+  version: dcc070983490608a14480e3bf943bad464785df5
 - name: github.com/fiorix/go-diameter
-  version: 16028e641c19a8dd67509053bc558d389258ff6d
+  version: abaf0a5b14a05f3a4a75b8fe23066ab1f898aeac
   subpackages:
   - diam
   - diam/avp
@@ -31,27 +44,39 @@ imports:
   - diam/sm
   - diam/sm/smparser
   - diam/sm/smpeer
-- name: github.com/ishidawataru/sctp
-  version: 6e2cb1366111dcf547c13531e3a263a067715847
+- name: github.com/fsnotify/fsnotify
+  version: ccc981bf80385c528a65fbfdd49bf2d8da22aa23
 - name: github.com/go-sql-driver/mysql
-  version: 99ff426eb706cffe92ff3d058e168b278cabf7c7
+  version: 6be42e0ff99645d7d9626d779001a46e39c5f280
+- name: github.com/go-stack/stack
+  version: 2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a
+- name: github.com/golang/snappy
+  version: 2e65f85255dbc3072edf28d6b5b8efc472979f5a
 - name: github.com/gorhill/cronexpr
   version: 88b0669f7d75f171bd612b874e52b95c190218df
+- name: github.com/ishidawataru/sctp
+  version: 6e2cb1366111dcf547c13531e3a263a067715847
 - name: github.com/jinzhu/gorm
-  version: 5be9bd34135805e0332b993378864b159784d8a8
+  version: 472c70caa40267cb89fd8facb07fe6454b578626
 - name: github.com/jinzhu/inflection
-  version: 1c35d901db3da928c72a72d8458480cc9ade058f
+  version: 04140366298a54a039076d798123ffa108fff46c
 - name: github.com/kr/pty
   version: db8e3cd836b82e82e0a9c8edc6896967dd31374f
 - name: github.com/lib/pq
-  version: 4ded0e9383f75c197b3a2aaa6d590ac52df6fd79
+  version: 9eb73efc1fcc404148b56765b0d3f61d9a5ef8ee
   subpackages:
   - oid
+- name: github.com/Masterminds/semver
+  version: c7af12943936e8c39859482e61f0574c2fd7fc75
+- name: github.com/mattn/go-runewidth
+  version: c88d7e5f2e24de48a200a2655ac8a0910be9a0f7
 - name: github.com/mediocregopher/radix.v2
-  version: 94360be262532d465b7e4760c7a67195d3319a87
+  version: b67df6e626f993b64b3ca9f4b8630900e61002e3
   subpackages:
   - pool
+  - pubsub
   - redis
+  - sentinel
 - name: github.com/mitchellh/mapstructure
   version: fa473d140ef3c6adf42d6b391fe76707f1f243c8
 - name: github.com/peterh/liner
@@ -120,18 +145,15 @@ imports:
 - name: github.com/Masterminds/semver
   version: c7af12943936e8c39859482e61f0574c2fd7fc75
 - name: github.com/mongodb/mongo-go-driver
-  version: 53671960ce090c91e7f647f04e09ed469ce22f09
+  version: 03d16bbe636981a408ff359412697fa9ff16ec83
   subpackages:
   - bson
   - bson/bsoncodec
-  - bson/bsoncore
   - bson/bsonrw
   - bson/bsontype
   - bson/decimal
-  - bson/elements
   - bson/objectid
-  - bson/parser
-  - bson/parser/ast
+  - bson/primitive
   - core/address
   - core/auth
   - core/auth/internal/gssapi
@@ -142,8 +164,6 @@ imports:
   - core/description
   - core/dispatch
   - core/event
-  - core/readconcern
-  - core/readpref
   - core/result
   - core/session
   - core/tag
@@ -151,20 +171,26 @@ imports:
   - core/uuid
   - core/version
   - core/wiremessage
-  - core/writeconcern
-  - examples/documentation_examples
   - internal
   - mongo
+  - mongo/readconcern
+  - mongo/readpref
+  - mongo/writeconcern
   - options
-- name: github.com/pmezard/go-difflib
-  version: 792786c7400a136282c1664665ae0a8db921c6c2
-  subpackages:
-  - difflib
+  - x/bsonx
+  - x/bsonx/bsoncore
+- name: github.com/peterh/liner
+  version: 5a0dfa99e2aa1d433a9642e863da51402e609376
+- name: github.com/streadway/amqp
+  version: 27835f1a64e97101d95306211f03c0620ffa295d
 - name: github.com/stretchr/testify
-  version: f35b8ab0b5a2cef36673838d662e249dd9c94686
+  version: 8019298d9fa5a04fc2ad10ae03349df3483096a6
   subpackages:
-  - assert
   - require
+- name: github.com/ugorji/go
+  version: ce1d12656603bc8fa8009023db97a0f0176ca4b9
+  subpackages:
+  - codec
 - name: github.com/xdg/scram
   version: 7eeb5667e42c09cb51bf7b7c28aea8c56767da90
 - name: github.com/xdg/stringprep
@@ -173,13 +199,45 @@ imports:
   version: 3d3f9f413869b949e48070b5bc593aa22cc2b8f2
   subpackages:
   - pbkdf2
+- name: golang.org/x/net
+  version: adae6a3d119ae4890b46832a2e88a95adc62b8e7
+  subpackages:
+  - context
+  - html
+  - html/atom
+  - html/charset
+  - websocket
 - name: golang.org/x/sync
   version: 42b317875d0fa942474b76e1b46a6060d720ae6e
   subpackages:
   - semaphore
-- name: golang.org/x/text
-  version: ceefd2213ed29504fff30155163c8f59827734f3
+- name: golang.org/x/sys
+  version: 93218def8b18e66adbdab3eca8ec334700329f1f
   subpackages:
+  - unix
+- name: golang.org/x/text
+  version: 6f44c5a2ea40ee3593d98cdcc905cc1fdaa660e2
+  subpackages:
+  - encoding
+  - encoding/charmap
+  - encoding/htmlindex
+  - encoding/internal
+  - encoding/internal/identifier
+  - encoding/japanese
+  - encoding/korean
+  - encoding/simplifiedchinese
+  - encoding/traditionalchinese
+  - encoding/unicode
+  - internal/language
+  - internal/language/compact
+  - internal/tag
+  - internal/utf8internal
+  - language
+  - runes
   - transform
   - unicode/norm
+- name: google.golang.org/appengine
+  version: 4a4468ece617fc8205e99368fa2200e9d1fad421
+  subpackages:
+  - cloudsql
 testImports: []

glide.yaml

@@ -46,8 +46,36 @@ import:
 - package: github.com/mongodb/mongo-go-driver
   subpackages:
   - bson
+  - bson/bsoncodec
+  - bson/bsonrw
+  - bson/bsontype
+  - bson/decimal
+  - bson/objectid
+  - bson/primitive
+  - core/address
+  - core/auth
+  - core/auth/internal/gssapi
+  - core/command
+  - core/compressor
+  - core/connection
+  - core/connstring
+  - core/description
+  - core/dispatch
+  - core/event
+  - core/result
+  - core/session
+  - core/tag
+  - core/topology
+  - core/uuid
+  - core/version
+  - core/wiremessage
+  - internal
   - mongo
+  - mongo/readconcern
+  - mongo/readpref
+  - mongo/writeconcern
   - options
+  - x/bsonx
 - package: github.com/stretchr/testify
   subpackages:
   - require