From 1ec59ad7944c68ce6db7fe2cbf9d51651f020ce3 Mon Sep 17 00:00:00 2001 From: DanB Date: Sat, 4 Apr 2015 16:35:59 +0200 Subject: [PATCH] Adding supplier field in StoredCdr and related, rename IgnoreDerived-> FilterOnDerived in getCdrs function, fixed postgres load from tp_rates --- data/storage/mysql/create_cdrs_tables.sql | 2 + data/storage/postgres/create_cdrs_tables.sql | 2 + engine/cdrs.go | 8 +- engine/cgrcdr.go | 1 + engine/cgrcdr_test.go | 5 +- engine/event.go | 1 + engine/models.go | 2 + engine/storage_mysql.go | 3 +- engine/storage_mysql_local_test.go | 31 +-- engine/storage_postgres.go | 4 +- engine/storage_psql_local_test.go | 2 +- engine/storage_sql.go | 201 +++++++++++++++---- engine/storedcdr.go | 13 +- engine/storedcdr_test.go | 20 +- general_tests/tutorial_fs_calls_test.go | 2 +- general_tests/tutorial_kam_calls_test.go | 2 +- utils/apitpdata.go | 54 ++++- utils/consts.go | 3 +- 18 files changed, 272 insertions(+), 84 deletions(-) diff --git a/data/storage/mysql/create_cdrs_tables.sql b/data/storage/mysql/create_cdrs_tables.sql index eee36efc8..e9264ea86 100644 --- a/data/storage/mysql/create_cdrs_tables.sql +++ b/data/storage/mysql/create_cdrs_tables.sql @@ -21,6 +21,7 @@ CREATE TABLE cdrs_primary ( setup_time datetime NOT NULL, answer_time datetime NOT NULL, `usage` DECIMAL(30,9) NOT NULL, + supplier varchar(128) NOT NULL, created_at TIMESTAMP, deleted_at TIMESTAMP, PRIMARY KEY (id), @@ -91,6 +92,7 @@ CREATE TABLE `rated_cdrs` ( setup_time datetime NOT NULL, answer_time datetime NOT NULL, `usage` DECIMAL(30,9) NOT NULL, + supplier varchar(128) NOT NULL, cost DECIMAL(20,4) DEFAULT NULL, extra_info text, created_at TIMESTAMP, diff --git a/data/storage/postgres/create_cdrs_tables.sql b/data/storage/postgres/create_cdrs_tables.sql index 5c4fbad15..e4e8b8fc5 100644 --- a/data/storage/postgres/create_cdrs_tables.sql +++ b/data/storage/postgres/create_cdrs_tables.sql @@ -21,6 +21,7 @@ CREATE TABLE cdrs_primary ( setup_time TIMESTAMP NOT NULL, answer_time TIMESTAMP NOT NULL, usage NUMERIC(30,9) NOT NULL, + supplier VARCHAR(128) NOT NULL, created_at TIMESTAMP, deleted_at TIMESTAMP, UNIQUE (cgrid) @@ -87,6 +88,7 @@ CREATE TABLE rated_cdrs ( setup_time TIMESTAMP NOT NULL, answer_time TIMESTAMP NOT NULL, usage NUMERIC(30,9) NOT NULL, + supplier VARCHAR(128) NOT NULL, cost NUMERIC(20,4) DEFAULT NULL, extra_info text, created_at TIMESTAMP, diff --git a/engine/cdrs.go b/engine/cdrs.go index 0c87836af..be4041905 100644 --- a/engine/cdrs.go +++ b/engine/cdrs.go @@ -111,9 +111,11 @@ func (self *CdrServer) RateCdrs(cgrIds, runIds, tors, cdrHosts, cdrSources, reqT } else if rerateRated { costStart = utils.Float64Pointer(0.0) } - cdrs, _, err := self.cdrDb.GetStoredCdrs(&utils.CdrsFilter{CgrIds: cgrIds, RunIds: runIds, Tors: tors, CdrHosts: cdrHosts, CdrSources: cdrSources, ReqTypes: reqTypes, Directions: directions, - Tenants: tenants, Categories: categories, Accounts: accounts, Subjects: subjects, DestPrefixes: destPrefixes, RatedAccounts: ratedAccounts, RatedSubjects: ratedSubjects, - OrderIdStart: orderIdStart, OrderIdEnd: orderIdEnd, AnswerTimeStart: &timeStart, AnswerTimeEnd: &timeEnd, CostStart: costStart, CostEnd: costEnd, IgnoreDerived: true}) + cdrs, _, err := self.cdrDb.GetStoredCdrs(&utils.CdrsFilter{CgrIds: cgrIds, RunIds: []string{utils.DEFAULT_RUNID}, Tors: tors, CdrHosts: cdrHosts, CdrSources: cdrSources, + ReqTypes: reqTypes, Directions: directions, Tenants: tenants, Categories: categories, Accounts: accounts, + Subjects: subjects, DestPrefixes: destPrefixes, RatedAccounts: ratedAccounts, RatedSubjects: ratedSubjects, + OrderIdStart: orderIdStart, OrderIdEnd: orderIdEnd, AnswerTimeStart: &timeStart, AnswerTimeEnd: &timeEnd, + CostStart: costStart, CostEnd: costEnd}) if err != nil { return err } diff --git a/engine/cgrcdr.go b/engine/cgrcdr.go index c49973c18..55cbf73db 100644 --- a/engine/cgrcdr.go +++ b/engine/cgrcdr.go @@ -71,6 +71,7 @@ func (cgrCdr CgrCdr) AsStoredCdr() *StoredCdr { storCdr.SetupTime, _ = utils.ParseTimeDetectLayout(cgrCdr[utils.SETUP_TIME]) // Not interested to process errors, should do them if necessary in a previous step storCdr.AnswerTime, _ = utils.ParseTimeDetectLayout(cgrCdr[utils.ANSWER_TIME]) storCdr.Usage, _ = utils.ParseDurationWithSecs(cgrCdr[utils.USAGE]) + storCdr.Supplier = cgrCdr[utils.SUPPLIER] storCdr.ExtraFields = cgrCdr.getExtraFields() storCdr.Cost = -1 return storCdr diff --git a/engine/cgrcdr_test.go b/engine/cgrcdr_test.go index fda8cfcfa..961de2896 100644 --- a/engine/cgrcdr_test.go +++ b/engine/cgrcdr_test.go @@ -38,14 +38,13 @@ func TestCgrCdrAsStoredCdr(t *testing.T) { utils.DIRECTION: utils.OUT, utils.TENANT: "cgrates.org", utils.CATEGORY: "call", utils.ACCOUNT: "1001", utils.SUBJECT: "1001", utils.DESTINATION: "1002", utils.SETUP_TIME: "2013-11-07T08:42:20Z", utils.ANSWER_TIME: "2013-11-07T08:42:26Z", - utils.USAGE: "10", - "field_extr1": "val_extr1", "fieldextr2": "valextr2"} + utils.USAGE: "10", utils.SUPPLIER: "SUPPL1", "field_extr1": "val_extr1", "fieldextr2": "valextr2"} setupTime, _ := utils.ParseTimeDetectLayout(cgrCdr["setup_time"]) expctRtCdr := &StoredCdr{CgrId: utils.Sha1(cgrCdr["accid"], setupTime.String()), TOR: utils.VOICE, AccId: cgrCdr["accid"], CdrHost: cgrCdr["cdrhost"], CdrSource: cgrCdr["cdrsource"], ReqType: cgrCdr["reqtype"], Direction: cgrCdr[utils.DIRECTION], Tenant: cgrCdr["tenant"], Category: cgrCdr[utils.CATEGORY], Account: cgrCdr["account"], Subject: cgrCdr["subject"], Destination: cgrCdr["destination"], SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), - Usage: time.Duration(10) * time.Second, + Usage: time.Duration(10) * time.Second, Supplier: "SUPPL1", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: -1} if storedCdr := cgrCdr.AsStoredCdr(); !reflect.DeepEqual(expctRtCdr, storedCdr) { t.Errorf("Expecting %v, received: %v", expctRtCdr, storedCdr) diff --git a/engine/event.go b/engine/event.go index a4748067a..c727d7380 100644 --- a/engine/event.go +++ b/engine/event.go @@ -40,6 +40,7 @@ type Event interface { GetAnswerTime(string) (time.Time, error) GetEndTime() (time.Time, error) GetDuration(string) (time.Duration, error) + //GetSupplier(string) string GetOriginatorIP(string) string GetExtraFields() map[string]string MissingParameter() bool diff --git a/engine/models.go b/engine/models.go index 1ebd09419..b29d8234d 100644 --- a/engine/models.go +++ b/engine/models.go @@ -299,6 +299,7 @@ type TblCdrsPrimary struct { SetupTime time.Time AnswerTime time.Time Usage float64 + Supplier string CreatedAt time.Time DeletedAt time.Time } @@ -356,6 +357,7 @@ type TblRatedCdr struct { SetupTime time.Time AnswerTime time.Time Usage float64 + Supplier string Cost float64 ExtraInfo string CreatedAt time.Time diff --git a/engine/storage_mysql.go b/engine/storage_mysql.go index 162614041..effa7a032 100644 --- a/engine/storage_mysql.go +++ b/engine/storage_mysql.go @@ -107,7 +107,7 @@ func (self *MySQLStorage) LogCallCost(cgrid, source, runid string, cc *CallCost) } func (self *MySQLStorage) SetRatedCdr(storedCdr *StoredCdr) (err error) { - _, err = self.Db.Exec(fmt.Sprintf("INSERT INTO %s (cgrid,runid,reqtype,direction,tenant,category,account,subject,destination,setup_time,answer_time,`usage`,cost,extra_info,created_at) VALUES ('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s',%v,%f,'%s','%s') ON DUPLICATE KEY UPDATE reqtype=values(reqtype),direction=values(direction),tenant=values(tenant),category=values(category),account=values(account),subject=values(subject),destination=values(destination),setup_time=values(setup_time),answer_time=values(answer_time),`usage`=values(`usage`),cost=values(cost),extra_info=values(extra_info), updated_at='%s'", + _, err = self.Db.Exec(fmt.Sprintf("INSERT INTO %s (cgrid,runid,reqtype,direction,tenant,category,account,subject,destination,setup_time,answer_time,`usage`,supplier,cost,extra_info,created_at) VALUES ('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s',%v,'%s',%f,'%s','%s') ON DUPLICATE KEY UPDATE reqtype=values(reqtype),direction=values(direction),tenant=values(tenant),category=values(category),account=values(account),subject=values(subject),destination=values(destination),setup_time=values(setup_time),answer_time=values(answer_time),`usage`=values(`usage`),cost=values(cost),supplier=values(supplier),extra_info=values(extra_info), updated_at='%s'", utils.TBL_RATED_CDRS, storedCdr.CgrId, storedCdr.MediationRunId, @@ -121,6 +121,7 @@ func (self *MySQLStorage) SetRatedCdr(storedCdr *StoredCdr) (err error) { storedCdr.SetupTime, storedCdr.AnswerTime, storedCdr.Usage.Seconds(), + storedCdr.Supplier, storedCdr.Cost, storedCdr.ExtraInfo, time.Now().Format(time.RFC3339), diff --git a/engine/storage_mysql_local_test.go b/engine/storage_mysql_local_test.go index adc586e9b..751a74722 100644 --- a/engine/storage_mysql_local_test.go +++ b/engine/storage_mysql_local_test.go @@ -433,22 +433,23 @@ func TestMySQLSetCdr(t *testing.T) { } cgrCdr1 := &CgrCdr{utils.TOR: utils.VOICE, utils.ACCID: "aaa1", utils.CDRHOST: "192.168.1.1", utils.REQTYPE: utils.META_RATED, utils.DIRECTION: "*out", utils.TENANT: "cgrates.org", utils.CATEGORY: "call", utils.ACCOUNT: "1001", utils.SUBJECT: "1001", utils.DESTINATION: "1002", utils.SETUP_TIME: "2013-11-08T08:42:20Z", - utils.ANSWER_TIME: "2013-11-08T08:42:26Z", utils.USAGE: "10s", "field_extr1": "val_extr1", "fieldextr2": "valextr2", utils.CDRSOURCE: TEST_SQL} + utils.ANSWER_TIME: "2013-11-08T08:42:26Z", utils.USAGE: "10s", utils.SUPPLIER: "SUPPL1", "field_extr1": "val_extr1", "fieldextr2": "valextr2", utils.CDRSOURCE: TEST_SQL} + cgrCdr2 := &CgrCdr{utils.TOR: utils.VOICE, utils.ACCID: "aaa2", utils.CDRHOST: "192.168.1.1", utils.REQTYPE: utils.META_PREPAID, utils.DIRECTION: "*out", utils.TENANT: "cgrates.org", utils.CATEGORY: "call", utils.ACCOUNT: "1001", utils.SUBJECT: "1001", utils.DESTINATION: "1002", utils.SETUP_TIME: "2013-11-08T08:42:22Z", - utils.ANSWER_TIME: "2013-11-08T08:42:26Z", utils.USAGE: "20", "field_extr1": "val_extr1", "fieldextr2": "valextr2", "cdrsource": TEST_SQL} + utils.ANSWER_TIME: "2013-11-08T08:42:26Z", utils.USAGE: "20", utils.SUPPLIER: "SUPPL1", "field_extr1": "val_extr1", "fieldextr2": "valextr2", "cdrsource": TEST_SQL} cgrCdr3 := &CgrCdr{utils.TOR: utils.VOICE, utils.ACCID: "aaa3", utils.CDRHOST: "192.168.1.1", utils.REQTYPE: utils.META_RATED, utils.DIRECTION: "*out", utils.TENANT: "cgrates.org", utils.CATEGORY: "premium_call", utils.ACCOUNT: "1002", utils.SUBJECT: "1002", utils.DESTINATION: "1001", utils.SETUP_TIME: "2013-11-07T08:42:24Z", - utils.ANSWER_TIME: "2013-11-07T08:42:26Z", utils.USAGE: "60s", "field_extr1": "val_extr1", "fieldextr2": "valextr2", "cdrsource": TEST_SQL} + utils.ANSWER_TIME: "2013-11-07T08:42:26Z", utils.USAGE: "60s", utils.SUPPLIER: "SUPPL1", "field_extr1": "val_extr1", "fieldextr2": "valextr2", "cdrsource": TEST_SQL} cgrCdr4 := &CgrCdr{utils.TOR: utils.VOICE, utils.ACCID: "aaa4", utils.CDRHOST: "192.168.1.2", utils.REQTYPE: utils.META_PSEUDOPREPAID, utils.DIRECTION: "*out", utils.TENANT: "itsyscom.com", utils.CATEGORY: "call", utils.ACCOUNT: "1001", utils.SUBJECT: "1001", utils.DESTINATION: "+4986517174964", utils.SETUP_TIME: "2013-11-07T08:42:21Z", - utils.ANSWER_TIME: "2013-11-07T08:42:26Z", utils.USAGE: "1m2s", "field_extr1": "val_extr1", "fieldextr2": "valextr2", "cdrsource": TEST_SQL} + utils.ANSWER_TIME: "2013-11-07T08:42:26Z", utils.USAGE: "1m2s", utils.SUPPLIER: "SUPPL1", "field_extr1": "val_extr1", "fieldextr2": "valextr2", "cdrsource": TEST_SQL} cgrCdr5 := &CgrCdr{utils.TOR: utils.VOICE, utils.ACCID: "aaa5", utils.CDRHOST: "192.168.1.2", utils.REQTYPE: utils.META_POSTPAID, utils.DIRECTION: "*out", utils.TENANT: "itsyscom.com", utils.CATEGORY: "call", utils.ACCOUNT: "1002", utils.SUBJECT: "1002", utils.DESTINATION: "+4986517174963", utils.SETUP_TIME: "2013-11-07T08:42:25Z", - utils.ANSWER_TIME: "2013-11-07T08:42:26Z", utils.USAGE: "15s", "field_extr1": "val_extr1", "fieldextr2": "valextr2", "cdrsource": TEST_SQL} + utils.ANSWER_TIME: "2013-11-07T08:42:26Z", utils.USAGE: "15s", utils.SUPPLIER: "SUPPL1", "field_extr1": "val_extr1", "fieldextr2": "valextr2", "cdrsource": TEST_SQL} for _, cdr := range []*CgrCdr{cgrCdr1, cgrCdr2, cgrCdr3, cgrCdr4, cgrCdr5} { if err := mysqlDb.SetCdr(cdr.AsStoredCdr()); err != nil { @@ -458,19 +459,22 @@ func TestMySQLSetCdr(t *testing.T) { strCdr1 := &StoredCdr{TOR: utils.VOICE, AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), - Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + Usage: time.Duration(10) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201} strCdr1.CgrId = utils.Sha1(strCdr1.AccId, strCdr1.SetupTime.String()) strCdr2 := &StoredCdr{TOR: utils.VOICE, AccId: "bbb2", CdrHost: "192.168.1.2", CdrSource: "UNKNOWN2", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), - Usage: time.Duration(12) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + Usage: time.Duration(12) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: utils.DEFAULT_RUNID, Cost: 0.201} strCdr2.CgrId = utils.Sha1(strCdr2.AccId, strCdr2.SetupTime.String()) strCdr3 := &StoredCdr{TOR: utils.VOICE, AccId: "bbb3", CdrHost: "192.168.1.1", CdrSource: TEST_SQL, ReqType: utils.META_RATED, Direction: "*out", Tenant: "itsyscom.com", Category: "call", Account: "1002", Subject: "1000", Destination: "+4986517174963", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), - Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + Usage: time.Duration(10) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201} strCdr3.CgrId = utils.Sha1(strCdr3.AccId, strCdr3.SetupTime.String()) @@ -488,19 +492,22 @@ func TestMySQLSetRatedCdr(t *testing.T) { strCdr1 := &StoredCdr{TOR: utils.VOICE, AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), - Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + Usage: time.Duration(10) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201} strCdr1.CgrId = utils.Sha1(strCdr1.AccId, strCdr1.SetupTime.String()) strCdr2 := &StoredCdr{TOR: utils.VOICE, AccId: "bbb2", CdrHost: "192.168.1.2", CdrSource: "UNKNOWN", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), - Usage: time.Duration(12) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + Usage: time.Duration(12) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: utils.DEFAULT_RUNID, Cost: 0.201} strCdr2.CgrId = utils.Sha1(strCdr2.AccId, strCdr2.SetupTime.String()) strCdr3 := &StoredCdr{TOR: utils.VOICE, AccId: "bbb3", CdrHost: "192.168.1.1", CdrSource: TEST_SQL, ReqType: utils.META_RATED, Direction: "*out", Tenant: "itsyscom.com", Category: "call", Account: "1002", Subject: "1002", Destination: "+4986517174964", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), - Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + Usage: time.Duration(10) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: "wholesale_run", Cost: 1.201} strCdr3.CgrId = utils.Sha1(strCdr3.AccId, strCdr3.SetupTime.String()) @@ -801,7 +808,7 @@ func TestMySQLGetStoredCdrs(t *testing.T) { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Filter on ignoreDerived - if storedCdrs, _, err := mysqlDb.GetStoredCdrs(&utils.CdrsFilter{AnswerTimeStart: &timeStart, AnswerTimeEnd: &timeEnd, IgnoreDerived: true}); err != nil { + if storedCdrs, _, err := mysqlDb.GetStoredCdrs(&utils.CdrsFilter{AnswerTimeStart: &timeStart, AnswerTimeEnd: &timeEnd, FilterOnDerived: true}); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 0 { // ToDo: Recheck this value t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) diff --git a/engine/storage_postgres.go b/engine/storage_postgres.go index f6d357eaf..1c8715ea5 100644 --- a/engine/storage_postgres.go +++ b/engine/storage_postgres.go @@ -138,6 +138,7 @@ func (self *PostgresStorage) SetRatedCdr(cdr *StoredCdr) (err error) { SetupTime: cdr.SetupTime, AnswerTime: cdr.AnswerTime, Usage: cdr.Usage.Seconds(), + Supplier: cdr.Supplier, Cost: cdr.Cost, ExtraInfo: cdr.ExtraInfo, CreatedAt: time.Now(), @@ -147,7 +148,8 @@ func (self *PostgresStorage) SetRatedCdr(cdr *StoredCdr) (err error) { tx = self.db.Begin() updated := tx.Model(TblRatedCdr{}).Where(&TblRatedCdr{Cgrid: cdr.CgrId, Runid: cdr.MediationRunId}).Updates(&TblRatedCdr{Reqtype: cdr.ReqType, Direction: cdr.Direction, Tenant: cdr.Tenant, Category: cdr.Category, Account: cdr.Account, Subject: cdr.Subject, Destination: cdr.Destination, - SetupTime: cdr.SetupTime, AnswerTime: cdr.AnswerTime, Usage: cdr.Usage.Seconds(), Cost: cdr.Cost, ExtraInfo: cdr.ExtraInfo, UpdatedAt: time.Now()}) + SetupTime: cdr.SetupTime, AnswerTime: cdr.AnswerTime, Usage: cdr.Usage.Seconds(), Supplier: cdr.Supplier, Cost: cdr.Cost, ExtraInfo: cdr.ExtraInfo, + UpdatedAt: time.Now()}) if updated.Error != nil { tx.Rollback() return updated.Error diff --git a/engine/storage_psql_local_test.go b/engine/storage_psql_local_test.go index a27145281..03be6a08f 100644 --- a/engine/storage_psql_local_test.go +++ b/engine/storage_psql_local_test.go @@ -803,7 +803,7 @@ func TestPSQLGetStoredCdrs(t *testing.T) { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Filter on ignoreDerived - if storedCdrs, _, err := psqlDb.GetStoredCdrs(&utils.CdrsFilter{AnswerTimeStart: &timeStart, AnswerTimeEnd: &timeEnd, IgnoreDerived: true}); err != nil { + if storedCdrs, _, err := psqlDb.GetStoredCdrs(&utils.CdrsFilter{AnswerTimeStart: &timeStart, AnswerTimeEnd: &timeEnd, FilterOnDerived: true}); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 0 { // ToDo: Recheck this value t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 5dffb0d72..f78759aca 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -739,6 +739,7 @@ func (self *SQLStorage) SetCdr(cdr *StoredCdr) error { SetupTime: cdr.SetupTime, AnswerTime: cdr.AnswerTime, Usage: cdr.Usage.Seconds(), + Supplier: cdr.Supplier, CreatedAt: time.Now()}) if saved.Error != nil { tx.Rollback() @@ -761,19 +762,19 @@ func (self *SQLStorage) GetStoredCdrs(qryFltr *utils.CdrsFilter) ([]*StoredCdr, var cdrs []*StoredCdr // Select string var selectStr string - if qryFltr.IgnoreDerived { // We use different tables to query account data in case of derived - selectStr = fmt.Sprintf("%s.cgrid,%s.id,%s.tor,%s.accid,%s.cdrhost,%s.cdrsource,%s.reqtype,%s.direction,%s.tenant,%s.category,%s.account,%s.subject,%s.destination,%s.setup_time,%s.answer_time,%s.usage,%s.extra_fields,%s.runid,%s.cost,%s.tor,%s.direction,%s.tenant,%s.category,%s.account,%s.subject,%s.destination,%s.cost,%s.timespans", - utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, + if qryFltr.FilterOnDerived { // We use different tables to query account data in case of derived + selectStr = fmt.Sprintf("%s.cgrid,%s.id,%s.tor,%s.accid,%s.cdrhost,%s.cdrsource,%s.reqtype,%s.direction,%s.tenant,%s.category,%s.account,%s.subject,%s.destination,%s.setup_time,%s.answer_time,%s.usage,%s.supplier,%s.extra_fields,%s.runid,%s.cost,%s.tor,%s.direction,%s.tenant,%s.category,%s.account,%s.subject,%s.destination,%s.cost,%s.timespans", + utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, + utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, + utils.TBL_CDRS_EXTRA, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, + utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS) + } else { + selectStr = fmt.Sprintf("%s.cgrid,%s.id,%s.tor,%s.accid,%s.cdrhost,%s.cdrsource,%s.reqtype,%s.direction,%s.tenant,%s.category,%s.account,%s.subject,%s.destination,%s.setup_time,%s.answer_time,%s.usage,%s.supplier,%s.extra_fields,%s.runid,%s.cost,%s.tor,%s.direction,%s.tenant,%s.category,%s.account,%s.subject,%s.destination,%s.cost,%s.timespans", utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, + utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_EXTRA, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS) - } else { - selectStr = fmt.Sprintf("%s.cgrid,%s.id,%s.tor,%s.accid,%s.cdrhost,%s.cdrsource,%s.reqtype,%s.direction,%s.tenant,%s.category,%s.account,%s.subject,%s.destination,%s.setup_time,%s.answer_time,%s.usage,%s.extra_fields,%s.runid,%s.cost,%s.tor,%s.direction,%s.tenant,%s.category,%s.account,%s.subject,%s.destination,%s.cost,%s.timespans", - utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_PRIMARY, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, - utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, - utils.TBL_CDRS_EXTRA, utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, - utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS, utils.TBL_COST_DETAILS) } // Join string joinStr := fmt.Sprintf("LEFT JOIN %s ON %s.cgrid=%s.cgrid LEFT JOIN %s ON %s.cgrid=%s.cgrid LEFT JOIN %s ON %s.cgrid=%s.cgrid AND %s.runid=%s.runid", utils.TBL_CDRS_EXTRA, utils.TBL_CDRS_PRIMARY, @@ -815,63 +816,133 @@ func (self *SQLStorage) GetStoredCdrs(qryFltr *utils.CdrsFilter) ([]*StoredCdr, q = q.Where(utils.TBL_CDRS_PRIMARY+".cdrsource not in (?)", qryFltr.NotCdrSources) } if len(qryFltr.ReqTypes) != 0 { - q = q.Where(utils.TBL_CDRS_PRIMARY+".reqtype in (?)", qryFltr.ReqTypes) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".reqtype in (?)", qryFltr.ReqTypes) } if len(qryFltr.NotReqTypes) != 0 { - q = q.Where(utils.TBL_CDRS_PRIMARY+".reqtype not in (?)", qryFltr.NotReqTypes) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".reqtype not in (?)", qryFltr.NotReqTypes) } if len(qryFltr.Directions) != 0 { - q = q.Where(utils.TBL_CDRS_PRIMARY+".direction in (?)", qryFltr.Directions) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".direction in (?)", qryFltr.Directions) } if len(qryFltr.NotDirections) != 0 { - q = q.Where(utils.TBL_CDRS_PRIMARY+".direction not in (?)", qryFltr.NotDirections) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".direction not in (?)", qryFltr.NotDirections) } if len(qryFltr.Tenants) != 0 { - q = q.Where(utils.TBL_CDRS_PRIMARY+".tenant in (?)", qryFltr.Tenants) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".tenant in (?)", qryFltr.Tenants) } if len(qryFltr.NotTenants) != 0 { - q = q.Where(utils.TBL_CDRS_PRIMARY+".tenant not in (?)", qryFltr.NotTenants) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".tenant not in (?)", qryFltr.NotTenants) } if len(qryFltr.Categories) != 0 { - q = q.Where(utils.TBL_CDRS_PRIMARY+".category in (?)", qryFltr.Categories) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".category in (?)", qryFltr.Categories) } if len(qryFltr.NotCategories) != 0 { - q = q.Where(utils.TBL_CDRS_PRIMARY+".category not in (?)", qryFltr.NotCategories) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".category not in (?)", qryFltr.NotCategories) } if len(qryFltr.Accounts) != 0 { - q = q.Where(utils.TBL_CDRS_PRIMARY+".account in (?)", qryFltr.Accounts) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".account in (?)", qryFltr.Accounts) } if len(qryFltr.NotAccounts) != 0 { - q = q.Where(utils.TBL_CDRS_PRIMARY+".account not in (?)", qryFltr.NotAccounts) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".account not in (?)", qryFltr.NotAccounts) } if len(qryFltr.Subjects) != 0 { - q = q.Where(utils.TBL_CDRS_PRIMARY+".subject in (?)", qryFltr.Subjects) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".subject in (?)", qryFltr.Subjects) } if len(qryFltr.NotSubjects) != 0 { - q = q.Where(utils.TBL_CDRS_PRIMARY+".subject not in (?)", qryFltr.NotSubjects) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".subject not in (?)", qryFltr.NotSubjects) } if len(qryFltr.DestPrefixes) != 0 { // A bit ugly but still more readable than scopes provided by gorm + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } qIds := bytes.NewBufferString("(") for idx, destPrefix := range qryFltr.DestPrefixes { if idx != 0 { qIds.WriteString(" OR") } - qIds.WriteString(fmt.Sprintf(" %s.destination LIKE '%s%%'", utils.TBL_CDRS_PRIMARY, destPrefix)) + qIds.WriteString(fmt.Sprintf(" %s.destination LIKE '%s%%'", tblName, destPrefix)) } qIds.WriteString(" )") q = q.Where(qIds.String()) } if len(qryFltr.NotDestPrefixes) != 0 { // A bit ugly but still more readable than scopes provided by gorm + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } qIds := bytes.NewBufferString("(") for idx, destPrefix := range qryFltr.NotDestPrefixes { if idx != 0 { qIds.WriteString(" AND") } - qIds.WriteString(fmt.Sprintf(" %s.destination not LIKE '%%%s%%'", utils.TBL_CDRS_PRIMARY, destPrefix)) + qIds.WriteString(fmt.Sprintf(" %s.destination not LIKE '%%%s%%'", tblName, destPrefix)) } qIds.WriteString(" )") q = q.Where(qIds.String()) } + if len(qryFltr.Suppliers) != 0 { + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".supplier in (?)", qryFltr.Subjects) + } + if len(qryFltr.NotSuppliers) != 0 { + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".supplier not in (?)", qryFltr.NotSubjects) + } if len(qryFltr.RatedAccounts) != 0 { q = q.Where(utils.TBL_COST_DETAILS+".account in (?)", qryFltr.RatedAccounts) } @@ -923,29 +994,76 @@ func (self *SQLStorage) GetStoredCdrs(qryFltr *utils.CdrsFilter) ([]*StoredCdr, q = q.Where(utils.TBL_CDRS_PRIMARY+".id < ?", qryFltr.OrderIdEnd) } if qryFltr.SetupTimeStart != nil { - q = q.Where(utils.TBL_CDRS_PRIMARY+".setup_time >= ?", qryFltr.SetupTimeStart) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".setup_time >= ?", qryFltr.SetupTimeStart) } if qryFltr.SetupTimeEnd != nil { - q = q.Where(utils.TBL_CDRS_PRIMARY+".setup_time < ?", qryFltr.SetupTimeEnd) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".setup_time < ?", qryFltr.SetupTimeEnd) } if qryFltr.AnswerTimeStart != nil && !qryFltr.AnswerTimeStart.IsZero() { // With IsZero we keep backwards compatible with ApierV1 - q = q.Where(utils.TBL_CDRS_PRIMARY+".answer_time >= ?", qryFltr.AnswerTimeStart) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".answer_time >= ?", qryFltr.AnswerTimeStart) } if qryFltr.AnswerTimeEnd != nil && !qryFltr.AnswerTimeEnd.IsZero() { - q = q.Where(utils.TBL_CDRS_PRIMARY+".answer_time < ?", qryFltr.AnswerTimeEnd) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".answer_time < ?", qryFltr.AnswerTimeEnd) + } + if qryFltr.CreatedAtStart != nil && !qryFltr.CreatedAtStart.IsZero() { // With IsZero we keep backwards compatible with ApierV1 + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".created_at >= ?", qryFltr.CreatedAtStart) + } + if qryFltr.CreatedAtEnd != nil && !qryFltr.CreatedAtEnd.IsZero() { + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".created_at < ?", qryFltr.CreatedAtEnd) + } + if qryFltr.UpdatedAtStart != nil && !qryFltr.UpdatedAtStart.IsZero() { // With IsZero we keep backwards compatible with ApierV1 + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".updated_at >= ?", qryFltr.UpdatedAtStart) + } + if qryFltr.UpdatedAtEnd != nil && !qryFltr.UpdatedAtEnd.IsZero() { + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".updated_at < ?", qryFltr.UpdatedAtEnd) } if qryFltr.UsageStart != nil { - q = q.Where(utils.TBL_CDRS_PRIMARY+".usage >= ?", qryFltr.UsageStart) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".usage >= ?", qryFltr.UsageStart) } if qryFltr.UsageEnd != nil { - q = q.Where(utils.TBL_CDRS_PRIMARY+".usage < ?", qryFltr.UsageEnd) - } - if qryFltr.RatedUsageStart != nil { - q = q.Where(utils.TBL_RATED_CDRS+".usage >= ?", qryFltr.RatedUsageStart) - } - if qryFltr.RatedUsageEnd != nil { - q = q.Where(utils.TBL_RATED_CDRS+".usage < ?", qryFltr.RatedUsageEnd) + tblName := utils.TBL_CDRS_PRIMARY + if qryFltr.FilterOnDerived { + tblName = utils.TBL_RATED_CDRS + } + q = q.Where(tblName+".usage < ?", qryFltr.UsageEnd) } + if qryFltr.CostStart != nil { if qryFltr.CostEnd == nil { q = q.Where(utils.TBL_RATED_CDRS+".cost >= ?", *qryFltr.CostStart) @@ -962,9 +1080,6 @@ func (self *SQLStorage) GetStoredCdrs(qryFltr *utils.CdrsFilter) ([]*StoredCdr, q = q.Where(fmt.Sprintf("( %s.cost IS NULL OR %s.cost < %f )", utils.TBL_RATED_CDRS, utils.TBL_RATED_CDRS, *qryFltr.CostEnd)) } } - if qryFltr.IgnoreDerived { - q = q.Where(utils.TBL_RATED_CDRS+".runid = ?", utils.DEFAULT_RUNID) - } if qryFltr.Paginator.Limit != nil { q = q.Limit(*qryFltr.Paginator.Limit) } @@ -986,14 +1101,16 @@ func (self *SQLStorage) GetStoredCdrs(qryFltr *utils.CdrsFilter) ([]*StoredCdr, return nil, 0, err } for rows.Next() { - var cgrid, tor, accid, cdrhost, cdrsrc, reqtype, direction, tenant, category, account, subject, destination, runid, ccTor, ccDirection, ccTenant, ccCategory, ccAccount, ccSubject, ccDestination sql.NullString + var cgrid, tor, accid, cdrhost, cdrsrc, reqtype, direction, tenant, category, account, subject, destination, runid, ccTor, + ccDirection, ccTenant, ccCategory, ccAccount, ccSubject, ccDestination, ccSupplier sql.NullString var extraFields, ccTimespansBytes []byte var setupTime, answerTime mysql.NullTime var orderid int64 var usage, cost, ccCost sql.NullFloat64 var extraFieldsMp map[string]string var ccTimespans TimeSpans - if err := rows.Scan(&cgrid, &orderid, &tor, &accid, &cdrhost, &cdrsrc, &reqtype, &direction, &tenant, &category, &account, &subject, &destination, &setupTime, &answerTime, &usage, + if err := rows.Scan(&cgrid, &orderid, &tor, &accid, &cdrhost, &cdrsrc, &reqtype, &direction, &tenant, &category, &account, &subject, &destination, + &setupTime, &answerTime, &usage, &ccSupplier, &extraFields, &runid, &cost, &ccTor, &ccDirection, &ccTenant, &ccCategory, &ccAccount, &ccSubject, &ccDestination, &ccCost, &ccTimespansBytes); err != nil { return nil, 0, err } @@ -1012,7 +1129,7 @@ func (self *SQLStorage) GetStoredCdrs(qryFltr *utils.CdrsFilter) ([]*StoredCdr, CgrId: cgrid.String, OrderId: orderid, TOR: tor.String, AccId: accid.String, CdrHost: cdrhost.String, CdrSource: cdrsrc.String, ReqType: reqtype.String, Direction: direction.String, Tenant: tenant.String, Category: category.String, Account: account.String, Subject: subject.String, Destination: destination.String, - SetupTime: setupTime.Time, AnswerTime: answerTime.Time, Usage: usageDur, + SetupTime: setupTime.Time, AnswerTime: answerTime.Time, Usage: usageDur, Supplier: ccSupplier.String, ExtraFields: extraFieldsMp, MediationRunId: runid.String, RatedAccount: ccAccount.String, RatedSubject: ccSubject.String, Cost: cost.Float64, } if ccTimespans != nil { @@ -1077,7 +1194,7 @@ func (self *SQLStorage) GetTpDestinations(tpid, tag string) (map[string]*Destina func (self *SQLStorage) GetTpRates(tpid, tag string) (map[string]*utils.TPRate, error) { rts := make(map[string]*utils.TPRate) var tpRates []TpRate - q := self.db.Where("tpid = ?", tpid) + q := self.db.Where("tpid = ?", tpid).Order("id") if len(tag) != 0 { q = q.Where("tag = ?", tag) } diff --git a/engine/storedcdr.go b/engine/storedcdr.go index 5374dc29e..218761550 100644 --- a/engine/storedcdr.go +++ b/engine/storedcdr.go @@ -33,9 +33,10 @@ import ( func NewStoredCdrFromExternalCdr(extCdr *ExternalCdr) (*StoredCdr, error) { var err error - storedCdr := &StoredCdr{CgrId: extCdr.CgrId, OrderId: extCdr.OrderId, TOR: extCdr.TOR, AccId: extCdr.AccId, CdrHost: extCdr.CdrHost, CdrSource: extCdr.CdrSource, ReqType: extCdr.ReqType, - Direction: extCdr.Direction, Tenant: extCdr.Tenant, Category: extCdr.Category, Account: extCdr.Account, Subject: extCdr.Subject, Destination: extCdr.Destination, - ExtraFields: extCdr.ExtraFields, MediationRunId: extCdr.MediationRunId, RatedAccount: extCdr.RatedAccount, RatedSubject: extCdr.RatedSubject, Cost: extCdr.Cost, Rated: extCdr.Rated} + storedCdr := &StoredCdr{CgrId: extCdr.CgrId, OrderId: extCdr.OrderId, TOR: extCdr.TOR, AccId: extCdr.AccId, CdrHost: extCdr.CdrHost, CdrSource: extCdr.CdrSource, + ReqType: extCdr.ReqType, Direction: extCdr.Direction, Tenant: extCdr.Tenant, Category: extCdr.Category, Account: extCdr.Account, Subject: extCdr.Subject, + Destination: extCdr.Destination, Supplier: extCdr.Supplier, ExtraFields: extCdr.ExtraFields, MediationRunId: extCdr.MediationRunId, + RatedAccount: extCdr.RatedAccount, RatedSubject: extCdr.RatedSubject, Cost: extCdr.Cost, Rated: extCdr.Rated} if storedCdr.SetupTime, err = utils.ParseTimeDetectLayout(extCdr.SetupTime); err != nil { return nil, err } @@ -71,6 +72,7 @@ type StoredCdr struct { SetupTime time.Time // set-up time of the event. Supported formats: datetime RFC3339 compatible, SQL datetime (eg: MySQL), unix timestamp. AnswerTime time.Time // answer time of the event. Supported formats: datetime RFC3339 compatible, SQL datetime (eg: MySQL), unix timestamp. Usage time.Duration // event usage information (eg: in case of tor=*voice this will represent the total duration of a call) + Supplier string // Supplier information when available ExtraFields map[string]string // Extra fields to be stored in CDR MediationRunId string RatedAccount string // Populated out of rating data @@ -154,6 +156,8 @@ func (storedCdr *StoredCdr) FieldAsString(rsrFld *utils.RSRField) string { return rsrFld.ParseValue(storedCdr.AnswerTime.Format(time.RFC3339)) case utils.USAGE: return strconv.FormatFloat(utils.Round(storedCdr.Usage.Seconds(), 0, utils.ROUNDING_MIDDLE), 'f', -1, 64) + case utils.SUPPLIER: + return rsrFld.ParseValue(storedCdr.Supplier) case utils.MEDI_RUNID: return rsrFld.ParseValue(storedCdr.MediationRunId) case utils.RATED_ACCOUNT: @@ -213,6 +217,7 @@ func (storedCdr *StoredCdr) AsHttpForm() url.Values { v.Set(utils.SETUP_TIME, storedCdr.SetupTime.Format(time.RFC3339)) v.Set(utils.ANSWER_TIME, storedCdr.AnswerTime.Format(time.RFC3339)) v.Set(utils.USAGE, storedCdr.FormatUsage(utils.SECONDS)) + v.Set(utils.SUPPLIER, storedCdr.Supplier) if storedCdr.CostDetails != nil { v.Set(utils.COST_DETAILS, storedCdr.CostDetailsJson()) } @@ -361,6 +366,7 @@ func (storedCdr *StoredCdr) AsExternalCdr() *ExternalCdr { SetupTime: storedCdr.SetupTime.Format(time.RFC3339), AnswerTime: storedCdr.AnswerTime.Format(time.RFC3339), Usage: storedCdr.FormatUsage(utils.SECONDS), + Supplier: storedCdr.Supplier, ExtraFields: storedCdr.ExtraFields, MediationRunId: storedCdr.MediationRunId, RatedAccount: storedCdr.RatedAccount, @@ -538,6 +544,7 @@ type ExternalCdr struct { SetupTime string AnswerTime string Usage string + Supplier string ExtraFields map[string]string MediationRunId string RatedAccount string diff --git a/engine/storedcdr_test.go b/engine/storedcdr_test.go index b1bc46c62..ce525c02e 100644 --- a/engine/storedcdr_test.go +++ b/engine/storedcdr_test.go @@ -34,13 +34,13 @@ func TestStoredCdrInterfaces(t *testing.T) { func TestNewStoredCdrFromExternalCdr(t *testing.T) { extCdr := &ExternalCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", - Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", + Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", Supplier: "SUPPL1", SetupTime: "2013-11-07T08:42:20Z", AnswerTime: "2013-11-07T08:42:26Z", MediationRunId: utils.DEFAULT_RUNID, Usage: "0.00000001", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dans", Rated: true, } eStorCdr := &StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", - Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", + Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", Supplier: "SUPPL1", SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, Usage: time.Duration(10), ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dans", Rated: true, } @@ -56,7 +56,7 @@ func TestFieldAsString(t *testing.T) { CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, - Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + Usage: time.Duration(10) * time.Second, Supplier: "SUPPL1", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dans", } if cdr.FieldAsString(&utils.RSRField{Id: utils.CGRID}) != cdr.CgrId || @@ -74,6 +74,7 @@ func TestFieldAsString(t *testing.T) { cdr.FieldAsString(&utils.RSRField{Id: utils.SETUP_TIME}) != cdr.SetupTime.Format(time.RFC3339) || cdr.FieldAsString(&utils.RSRField{Id: utils.ANSWER_TIME}) != cdr.AnswerTime.Format(time.RFC3339) || cdr.FieldAsString(&utils.RSRField{Id: utils.USAGE}) != "10" || + cdr.FieldAsString(&utils.RSRField{Id: utils.SUPPLIER}) != cdr.Supplier || cdr.FieldAsString(&utils.RSRField{Id: utils.MEDI_RUNID}) != cdr.MediationRunId || cdr.FieldAsString(&utils.RSRField{Id: utils.COST}) != "1.01" || cdr.FieldAsString(&utils.RSRField{Id: utils.RATED_ACCOUNT}) != "dan" || @@ -97,6 +98,7 @@ func TestFieldAsString(t *testing.T) { cdr.FieldAsString(&utils.RSRField{Id: utils.SETUP_TIME}) != cdr.SetupTime.Format(time.RFC3339), cdr.FieldAsString(&utils.RSRField{Id: utils.ANSWER_TIME}) != cdr.AnswerTime.Format(time.RFC3339), cdr.FieldAsString(&utils.RSRField{Id: utils.USAGE}) != "10", + cdr.FieldAsString(&utils.RSRField{Id: utils.SUPPLIER}) != cdr.Supplier, cdr.FieldAsString(&utils.RSRField{Id: utils.MEDI_RUNID}) != cdr.MediationRunId, cdr.FieldAsString(&utils.RSRField{Id: utils.RATED_ACCOUNT}) != "dan", cdr.FieldAsString(&utils.RSRField{Id: utils.RATED_SUBJECT}) != "dans", @@ -272,7 +274,8 @@ func TestStoredCdrAsHttpForm(t *testing.T) { CdrHost: "192.168.1.1", CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, - Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, RatedSubject: "dans", Cost: 1.01, + Usage: time.Duration(10) * time.Second, Supplier: "SUPPL1", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, RatedSubject: "dans", Cost: 1.01, } cdrForm := storCdr.AsHttpForm() if cdrForm.Get(utils.TOR) != utils.VOICE { @@ -317,6 +320,9 @@ func TestStoredCdrAsHttpForm(t *testing.T) { if cdrForm.Get(utils.USAGE) != "10" { t.Errorf("Expected: %s, received: %s", "10", cdrForm.Get(utils.USAGE)) } + if cdrForm.Get(utils.SUPPLIER) != "SUPPL1" { + t.Errorf("Expected: %s, received: %s", "1001", cdrForm.Get(utils.SUPPLIER)) + } if cdrForm.Get("field_extr1") != "val_extr1" { t.Errorf("Expected: %s, received: %s", "val_extr1", cdrForm.Get("field_extr1")) } @@ -422,13 +428,15 @@ func TestStoredCdrAsExternalCdr(t *testing.T) { AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID, - Usage: time.Duration(10), ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dans", + Usage: time.Duration(10), Supplier: "SUPPL1", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + Cost: 1.01, RatedAccount: "dan", RatedSubject: "dans", } expectOutCdr := &ExternalCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: utils.UNIT_TEST, ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: "2013-11-07T08:42:20Z", AnswerTime: "2013-11-07T08:42:26Z", MediationRunId: utils.DEFAULT_RUNID, - Usage: "0.00000001", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dans", + Usage: "0.00000001", Supplier: "SUPPL1", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, + Cost: 1.01, RatedAccount: "dan", RatedSubject: "dans", } if cdrOut := storCdr.AsExternalCdr(); !reflect.DeepEqual(expectOutCdr, cdrOut) { t.Errorf("Expected: %+v, received: %+v", expectOutCdr, cdrOut) diff --git a/general_tests/tutorial_fs_calls_test.go b/general_tests/tutorial_fs_calls_test.go index abe68d3db..9e98bad83 100644 --- a/general_tests/tutorial_fs_calls_test.go +++ b/general_tests/tutorial_fs_calls_test.go @@ -482,7 +482,7 @@ func TestTutFsCallsCdrs(t *testing.T) { t.Errorf("Unexpected Usage for CDR: %+v", reply[0]) } } - req = utils.RpcCdrsFilter{Accounts: []string{"1001"}, RunIds: []string{"derived_run1"}} + req = utils.RpcCdrsFilter{Accounts: []string{"1001"}, RunIds: []string{"derived_run1"}, FilterOnDerived: true} if err := tutFsCallsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) } else if len(reply) != 1 { diff --git a/general_tests/tutorial_kam_calls_test.go b/general_tests/tutorial_kam_calls_test.go index 601cc80a5..ab65f55ad 100644 --- a/general_tests/tutorial_kam_calls_test.go +++ b/general_tests/tutorial_kam_calls_test.go @@ -241,7 +241,7 @@ func TestTutKamCallsCdrs1001(t *testing.T) { t.Errorf("Unexpected Usage for CDR: %+v", reply[0].Usage) } } - req = utils.RpcCdrsFilter{Accounts: []string{"1001"}, RunIds: []string{"derived_run1"}} + req = utils.RpcCdrsFilter{Accounts: []string{"1001"}, RunIds: []string{"derived_run1"}, FilterOnDerived: true} if err := tutKamCallsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil { t.Error("Unexpected error: ", err.Error()) } else if len(reply) != 1 { diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 944fbe5ff..4c45931c2 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -914,6 +914,8 @@ type CdrsFilter struct { NotSubjects []string // Filter out specific subjects DestPrefixes []string // If provided, it will filter on destination prefix NotDestPrefixes []string // Filter out specific destination prefixes + Suppliers []string // If provided, it will filter the supplier + NotSuppliers []string // Filter out specific suppliers RatedAccounts []string // If provided, it will filter ratedaccount NotRatedAccounts []string // Filter out specific RatedAccounts RatedSubjects []string // If provided, it will filter the ratedsubject @@ -928,13 +930,15 @@ type CdrsFilter struct { SetupTimeEnd *time.Time // End interval, smaller than setupTime AnswerTimeStart *time.Time // Start of interval, bigger or equal than configured AnswerTimeEnd *time.Time // End interval, smaller than answerTime + CreatedAtStart *time.Time // Start of interval, bigger or equal than configured + CreatedAtEnd *time.Time // End interval, smaller than + UpdatedAtStart *time.Time // Start of interval, bigger or equal than configured + UpdatedAtEnd *time.Time // End interval, smaller than UsageStart *float64 // Start of the usage interval (>=) UsageEnd *float64 // End of the usage interval (<) - RatedUsageStart *float64 // Start of the ratedUsage interval (>=) - RatedUsageEnd *float64 // End of the ratedUsage interval (<) CostStart *float64 // Start of the cost interval (>=) CostEnd *float64 // End of the usage interval (<) - IgnoreDerived bool // Do not consider derived CDRs but original one + FilterOnDerived bool // Do not consider derived CDRs but original one Count bool // If true count the items instead of returning data Paginator } @@ -965,6 +969,8 @@ type RpcCdrsFilter struct { NotSubjects []string // Filter out specific subjects DestPrefixes []string // If provided, it will filter on destination prefix NotDestPrefixes []string // Filter out specific destination prefixes + Suppliers []string // If provided, it will filter the supplier + NotSuppliers []string // Filter out specific suppliers RatedAccounts []string // If provided, it will filter ratedaccount NotRatedAccounts []string // Filter out specific RatedAccounts RatedSubjects []string // If provided, it will filter the ratedsubject @@ -979,13 +985,15 @@ type RpcCdrsFilter struct { SetupTimeEnd string // End interval, smaller than setupTime AnswerTimeStart string // Start of interval, bigger or equal than configured AnswerTimeEnd string // End interval, smaller than answerTime + CreatedAtStart string // Start of interval, bigger or equal than configured + CreatedAtEnd string // End interval, smaller than + UpdatedAtStart string // Start of interval, bigger or equal than configured + UpdatedAtEnd string // End interval, smaller than UsageStart *float64 // Start of the usage interval (>=) UsageEnd *float64 // End of the usage interval (<) - RatedUsageStart *float64 // Start of the ratedUsage interval (>=) - RatedUsageEnd *float64 // End of the ratedUsage interval (<) CostStart *float64 // Start of the cost interval (>=) CostEnd *float64 // End of the usage interval (<) - IgnoreDerived bool // Do not consider derived CDRs but original one + FilterOnDerived bool // Do not consider derived CDRs but original one Paginator // Add pagination } @@ -1015,6 +1023,8 @@ func (self *RpcCdrsFilter) AsCdrsFilter() (*CdrsFilter, error) { NotSubjects: self.NotSubjects, DestPrefixes: self.DestPrefixes, NotDestPrefixes: self.NotDestPrefixes, + Suppliers: self.Suppliers, + NotSuppliers: self.NotSuppliers, RatedAccounts: self.RatedAccounts, NotRatedAccounts: self.NotRatedAccounts, RatedSubjects: self.RatedSubjects, @@ -1027,11 +1037,9 @@ func (self *RpcCdrsFilter) AsCdrsFilter() (*CdrsFilter, error) { OrderIdEnd: self.OrderIdEnd, UsageStart: self.UsageStart, UsageEnd: self.UsageEnd, - RatedUsageStart: self.RatedUsageStart, - RatedUsageEnd: self.RatedUsageEnd, CostStart: self.CostStart, CostEnd: self.CostEnd, - IgnoreDerived: self.IgnoreDerived, + FilterOnDerived: self.FilterOnDerived, Paginator: self.Paginator, } if len(self.SetupTimeStart) != 0 { @@ -1062,5 +1070,33 @@ func (self *RpcCdrsFilter) AsCdrsFilter() (*CdrsFilter, error) { cdrFltr.AnswerTimeEnd = &aTimeEnd } } + if len(self.CreatedAtStart) != 0 { + if tStart, err := ParseTimeDetectLayout(self.CreatedAtStart); err != nil { + return nil, err + } else { + cdrFltr.CreatedAtStart = &tStart + } + } + if len(self.CreatedAtEnd) != 0 { + if tEnd, err := ParseTimeDetectLayout(self.CreatedAtEnd); err != nil { + return nil, err + } else { + cdrFltr.CreatedAtEnd = &tEnd + } + } + if len(self.UpdatedAtStart) != 0 { + if tStart, err := ParseTimeDetectLayout(self.UpdatedAtStart); err != nil { + return nil, err + } else { + cdrFltr.UpdatedAtStart = &tStart + } + } + if len(self.UpdatedAtEnd) != 0 { + if tEnd, err := ParseTimeDetectLayout(self.UpdatedAtEnd); err != nil { + return nil, err + } else { + cdrFltr.UpdatedAtEnd = &tEnd + } + } return cdrFltr, nil } diff --git a/utils/consts.go b/utils/consts.go index cd3c53947..37a5c9014 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -104,6 +104,7 @@ const ( SETUP_TIME = "setup_time" ANSWER_TIME = "answer_time" USAGE = "usage" + SUPPLIER = "supplier" MEDI_RUNID = "mediation_runid" RATED_ACCOUNT = "rated_account" RATED_SUBJECT = "rated_subject" @@ -184,5 +185,5 @@ const ( var ( CdreCdrFormats = []string{CSV, DRYRUN, CDRE_FIXED_WIDTH} - PrimaryCdrFields = []string{TOR, ACCID, CDRHOST, CDRSOURCE, REQTYPE, DIRECTION, TENANT, CATEGORY, ACCOUNT, SUBJECT, DESTINATION, SETUP_TIME, ANSWER_TIME, USAGE} + PrimaryCdrFields = []string{TOR, ACCID, CDRHOST, CDRSOURCE, REQTYPE, DIRECTION, TENANT, CATEGORY, ACCOUNT, SUBJECT, DESTINATION, SETUP_TIME, ANSWER_TIME, USAGE, SUPPLIER} )