From 4cf69dd5520412ebd1465e9e0d288c0cabb43d68 Mon Sep 17 00:00:00 2001 From: adragusin Date: Mon, 9 Dec 2019 16:41:24 +0200 Subject: [PATCH 1/2] uniformed the error for duplicate key --- apier/v1/cdrs_it_test.go | 21 +++++++++ data/conf/samples/cdrsv1mongo/cgrates.json | 44 +++++++++++++++++++ data/conf/samples/cdrsv1mysql/cgrates.json | 41 +++++++++++++++++ data/conf/samples/cdrsv1postgres/cgrates.json | 43 ++++++++++++++++++ engine/storage_mongo_stordb.go | 3 ++ engine/storage_sql.go | 3 ++ 6 files changed, 155 insertions(+) create mode 100644 data/conf/samples/cdrsv1mongo/cgrates.json create mode 100644 data/conf/samples/cdrsv1mysql/cgrates.json create mode 100644 data/conf/samples/cdrsv1postgres/cgrates.json diff --git a/apier/v1/cdrs_it_test.go b/apier/v1/cdrs_it_test.go index a07f81809..1d19abb6d 100644 --- a/apier/v1/cdrs_it_test.go +++ b/apier/v1/cdrs_it_test.go @@ -55,6 +55,27 @@ func TestCDRsITInternal(t *testing.T) { } } +func TestCDRsITMongo(t *testing.T) { + cdrsConfDIR = "cdrsv1mongo" + for _, stest := range sTestsCDRsIT { + t.Run(cdrsConfDIR, stest) + } +} + +func TestCDRsITMySql(t *testing.T) { + cdrsConfDIR = "cdrsv1mysql" + for _, stest := range sTestsCDRsIT { + t.Run(cdrsConfDIR, stest) + } +} + +func TestCDRsITPostgres(t *testing.T) { + cdrsConfDIR = "cdrsv1postgres" + for _, stest := range sTestsCDRsIT { + t.Run(cdrsConfDIR, stest) + } +} + func testV1CDRsInitConfig(t *testing.T) { var err error cdrsCfgPath = path.Join(*dataDir, "conf", "samples", cdrsConfDIR) diff --git a/data/conf/samples/cdrsv1mongo/cgrates.json b/data/conf/samples/cdrsv1mongo/cgrates.json new file mode 100644 index 000000000..01b81d9c4 --- /dev/null +++ b/data/conf/samples/cdrsv1mongo/cgrates.json @@ -0,0 +1,44 @@ +{ +// CGRateS Configuration file +// +// Used in apier/v1/cdrs_it_test + + +"data_db": { + "db_type": "mongo", + "db_name": "10", + "db_port": 27017, +}, + + +"stor_db": { + "db_type": "mongo", + "db_name": "cgrates", + "db_port": 27017, +}, + +"rals": { + "enabled": true +}, + + +"scheduler": { + "enabled": true +}, + + +"cdrs": { + "enabled": true, + "rals_conns": [ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], +}, + + +"apier": { + "scheduler_conns": [ // connections to SchedulerS for reloads + {"address": "*internal"}, + ], +}, + +} diff --git a/data/conf/samples/cdrsv1mysql/cgrates.json b/data/conf/samples/cdrsv1mysql/cgrates.json new file mode 100644 index 000000000..10ad516c9 --- /dev/null +++ b/data/conf/samples/cdrsv1mysql/cgrates.json @@ -0,0 +1,41 @@ +{ +// CGRateS Configuration file +// +// Used in apier/v1/cdrs_it_test + + +"data_db": { // database used to store runtime data (eg: accounts, cdr stats) + "db_type": "redis", // data_db type: + "db_port": 6379, // data_db port to reach the database + "db_name": "10", // data_db database name to connect to +}, + +"stor_db": { + "db_password": "CGRateS.org", +}, + +"rals": { + "enabled": true +}, + + +"scheduler": { + "enabled": true +}, + + +"cdrs": { + "enabled": true, + "rals_conns": [ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], +}, + + +"apier": { + "scheduler_conns": [ // connections to SchedulerS for reloads + {"address": "*internal"}, + ], +}, + +} diff --git a/data/conf/samples/cdrsv1postgres/cgrates.json b/data/conf/samples/cdrsv1postgres/cgrates.json new file mode 100644 index 000000000..d1ac60b9d --- /dev/null +++ b/data/conf/samples/cdrsv1postgres/cgrates.json @@ -0,0 +1,43 @@ +{ +// CGRateS Configuration file +// +// Used in apier/v1/cdrs_it_test + + +"data_db": { // database used to store runtime data (eg: accounts, cdr stats) + "db_type": "redis", // data_db type: + "db_port": 6379, // data_db port to reach the database + "db_name": "10", // data_db database name to connect to +}, + +"stor_db": { + "db_type": "postgres", // stor database type to use: + "db_port": 5432, // the port to reach the stordb + "db_password": "CGRateS.org", +}, + +"rals": { + "enabled": true +}, + + +"scheduler": { + "enabled": true +}, + + +"cdrs": { + "enabled": true, + "rals_conns": [ + {"address": "127.0.0.1:2012", "transport":"*json"}, + ], +}, + + +"apier": { + "scheduler_conns": [ // connections to SchedulerS for reloads + {"address": "*internal"}, + ], +}, + +} diff --git a/engine/storage_mongo_stordb.go b/engine/storage_mongo_stordb.go index c490f9929..d9d024177 100644 --- a/engine/storage_mongo_stordb.go +++ b/engine/storage_mongo_stordb.go @@ -943,6 +943,9 @@ func (ms *MongoStorage) SetCDR(cdr *CDR, allowUpdate bool) (err error) { // return err } else { _, err = ms.getCol(ColCDRs).InsertOne(sctx, cdr) + if err != nil && strings.Contains(err.Error(), "E11000") { // Mongo returns E11000 when key is duplicated + err = utils.ErrExists + } } return err }) diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 1b59c35ae..992533dfa 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -879,6 +879,9 @@ func (self *SQLStorage) SetCDR(cdr *CDR, allowUpdate bool) error { if saved.Error != nil { tx.Rollback() if !allowUpdate { + if strings.Contains(saved.Error.Error(), "1062") || strings.Contains(saved.Error.Error(), "duplicate key") { // returns 1062/pq when key is duplicated + return utils.ErrExists + } return saved.Error } tx = self.db.Begin() From 076cdc48d2b8e62b350c0564ee061d70421b1af0 Mon Sep 17 00:00:00 2001 From: adragusin Date: Tue, 10 Dec 2019 10:00:05 +0200 Subject: [PATCH 2/2] Added test for duplicate key --- engine/storage_mongo_stordb.go | 15 +++++++-------- engine/stordb_it_test.go | 14 +++++++------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/engine/storage_mongo_stordb.go b/engine/storage_mongo_stordb.go index d9d024177..648c5eea0 100644 --- a/engine/storage_mongo_stordb.go +++ b/engine/storage_mongo_stordb.go @@ -931,7 +931,7 @@ func (ms *MongoStorage) RemoveSMCosts(qryFltr *utils.SMCostFilter) error { }) } -func (ms *MongoStorage) SetCDR(cdr *CDR, allowUpdate bool) (err error) { +func (ms *MongoStorage) SetCDR(cdr *CDR, allowUpdate bool) error { if cdr.OrderID == 0 { cdr.OrderID = ms.cnter.Next() } @@ -940,14 +940,13 @@ func (ms *MongoStorage) SetCDR(cdr *CDR, allowUpdate bool) (err error) { _, err = ms.getCol(ColCDRs).UpdateOne(sctx, bson.M{CGRIDLow: cdr.CGRID, RunIDLow: cdr.RunID}, bson.M{"$set": cdr}, options.Update().SetUpsert(true)) - // return err - } else { - _, err = ms.getCol(ColCDRs).InsertOne(sctx, cdr) - if err != nil && strings.Contains(err.Error(), "E11000") { // Mongo returns E11000 when key is duplicated - err = utils.ErrExists - } + return } - return err + _, err = ms.getCol(ColCDRs).InsertOne(sctx, cdr) + if err != nil && strings.Contains(err.Error(), "E11000") { // Mongo returns E11000 when key is duplicated + err = utils.ErrExists + } + return }) } diff --git a/engine/stordb_it_test.go b/engine/stordb_it_test.go index bf5d81dbb..0e008cce2 100644 --- a/engine/stordb_it_test.go +++ b/engine/stordb_it_test.go @@ -32,9 +32,8 @@ import ( ) var ( - cfg *config.CGRConfig - storDB StorDB - storDB2ndDBname string + cfg *config.CGRConfig + storDB StorDB ) // subtests to be executed for each confDIR @@ -71,7 +70,6 @@ func TestStorDBitMySQL(t *testing.T) { cfg.StorDbCfg().ConnMaxLifetime); err != nil { t.Fatal(err) } - storDB2ndDBname = "mysql" for _, stest := range sTestsStorDBit { stestFullName := runtime.FuncForPC(reflect.ValueOf(stest).Pointer()).Name() split := strings.Split(stestFullName, ".") @@ -91,7 +89,6 @@ func TestStorDBitPostgresSQL(t *testing.T) { cfg.StorDbCfg().MaxIdleConns, cfg.StorDbCfg().ConnMaxLifetime); err != nil { t.Fatal(err) } - storDB2ndDBname = "postgres" for _, stest := range sTestsStorDBit { stestFullName := runtime.FuncForPC(reflect.ValueOf(stest).Pointer()).Name() split := strings.Split(stestFullName, ".") @@ -110,7 +107,6 @@ func TestStorDBitMongo(t *testing.T) { utils.StorDB, cfg.StorDbCfg().StringIndexedFields, false); err != nil { t.Fatal(err) } - storDB2ndDBname = "todo" for _, stest := range sTestsStorDBit { stestFullName := runtime.FuncForPC(reflect.ValueOf(stest).Pointer()).Name() split := strings.Split(stestFullName, ".") @@ -1229,7 +1225,11 @@ func testStorDBitCRUDCDRs(t *testing.T) { t.Error(err) } } - + for _, cdr := range snd { + if err := storDB.SetCDR(cdr, false); err == nil || err != utils.ErrExists { + t.Error(err) // for mongo will fail because of indexes + } + } // READ if rcv, _, err := storDB.GetCDRs(&filter, false); err != nil { t.Error(err)