From a3fff4233865107e9f287ae878fa249e21e57940 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 8 Apr 2014 17:51:18 +0200 Subject: [PATCH] FsCgrId -> Sha1, making the CgrId even more uniquely by hashing it with setup time --- apier/apier_local_test.go | 7 ++++- apier/cdre.go | 2 +- cdrc/cdrc.go | 2 +- cdrc/cdrc_test.go | 2 +- cdre/csv.go | 15 +++++----- cdre/csv_test.go | 4 +-- cdre/fixedwidth.go | 26 ++++++++--------- cdre/fixedwidth_test.go | 8 +++--- cdrs/fscdr.go | 8 ++---- cdrs/fscdr_test.go | 8 +++--- charging_tests/ddazmbl1_test.go | 2 +- engine/action_timing.go | 6 ++-- engine/rateinterval.go | 6 ++-- engine/storage_interface.go | 4 +-- engine/storage_map.go | 8 +++--- engine/storage_mongo.go | 8 +++--- engine/storage_redis.go | 8 +++--- engine/storage_sql.go | 13 ++++----- engine/storage_sql_local_test.go | 49 +++++++++++++++++++++++--------- mediator/fsfilecsvcdr.go | 7 +++-- mediator/mediator.go | 2 +- mediator/mediator_rpc.go | 2 +- sessionmanager/event.go | 1 + sessionmanager/fsevent.go | 3 ++ sessionmanager/session.go | 6 ++-- utils/apitpdata.go | 16 +++++------ utils/cgrcdr.go | 8 ++---- utils/cgrcdr_test.go | 14 ++++++--- utils/coreutils.go | 12 ++++---- utils/rawcdr.go | 2 +- utils/storedcdr.go | 2 +- utils/storedcdr_test.go | 31 +++++++++++++------- 32 files changed, 166 insertions(+), 126 deletions(-) diff --git a/apier/apier_local_test.go b/apier/apier_local_test.go index 160c3c52b..937f49518 100644 --- a/apier/apier_local_test.go +++ b/apier/apier_local_test.go @@ -1322,9 +1322,11 @@ func TestCdrServer(t *testing.T) { httpClient := new(http.Client) cdrForm1 := url.Values{"accid": []string{"dsafdsaf"}, "cdrhost": []string{"192.168.1.1"}, "reqtype": []string{"rated"}, "direction": []string{"*out"}, "tenant": []string{"cgrates.org"}, "tor": []string{"call"}, "account": []string{"1001"}, "subject": []string{"1001"}, "destination": []string{"1002"}, + "setup_time": []string{"2013-11-07T08:42:22Z"}, "answer_time": []string{"2013-11-07T08:42:26Z"}, "duration": []string{"10"}, "field_extr1": []string{"val_extr1"}, "fieldextr2": []string{"valextr2"}} cdrForm2 := url.Values{"accid": []string{"adsafdsaf"}, "cdrhost": []string{"192.168.1.1"}, "reqtype": []string{"rated"}, "direction": []string{"*out"}, "tenant": []string{"cgrates.org"}, "tor": []string{"call"}, "account": []string{"1001"}, "subject": []string{"1001"}, "destination": []string{"1002"}, + "setup_time": []string{"2013-11-07T08:42:23Z"}, "answer_time": []string{"2013-11-07T08:42:26Z"}, "duration": []string{"10"}, "field_extr1": []string{"val_extr1"}, "fieldextr2": []string{"valextr2"}} for _, cdrForm := range []url.Values{cdrForm1, cdrForm2} { cdrForm.Set(utils.CDRSOURCE, engine.TEST_SQL) @@ -1344,7 +1346,10 @@ func TestExportCdrsToFile(t *testing.T) { t.Error("Failed to detect missing parameter") } req.CdrFormat = utils.CDRE_DRYRUN - expectReply := &utils.ExportedFileCdrs{ExportedFilePath: utils.CDRE_DRYRUN, TotalRecords: 2, ExportedCgrIds: []string{utils.FSCgrId("dsafdsaf"), utils.FSCgrId("adsafdsaf")}} + tm1, _ := utils.ParseTimeDetectLayout("2013-11-07T08:42:22Z") + tm2, _ := utils.ParseTimeDetectLayout("2013-11-07T08:42:23Z") + expectReply := &utils.ExportedFileCdrs{ExportedFilePath: utils.CDRE_DRYRUN, TotalRecords: 2, ExportedCgrIds: []string{utils.Sha1("dsafdsaf", tm1.String()), + utils.Sha1("adsafdsaf", tm2.String())}} if err := rater.Call("ApierV1.ExportCdrsToFile", req, &reply); err != nil { t.Error(err.Error()) } else if !reflect.DeepEqual(reply, expectReply) { diff --git a/apier/cdre.go b/apier/cdre.go index 7ae7ec48e..3d9bc0422 100644 --- a/apier/cdre.go +++ b/apier/cdre.go @@ -114,7 +114,7 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E } } csvWriter.Close() - *reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), ExportedCgrIds: exportedIds, UnexportedCgrIds: unexportedIds, + *reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), ExportedCgrIds: exportedIds, UnexportedCgrIds: unexportedIds, FirstOrderId: csvWriter.FirstOrderId(), LastOrderId: csvWriter.LastOrderId()} case utils.CDRE_FIXED_WIDTH: if len(fileName) == 0 { diff --git a/cdrc/cdrc.go b/cdrc/cdrc.go index 1c5cbc034..a12e992c5 100644 --- a/cdrc/cdrc.go +++ b/cdrc/cdrc.go @@ -138,7 +138,6 @@ func (self *Cdrc) recordAsStoredCdr(record []string) (*utils.StoredCdr, error) { } switch cfgFieldName { case utils.ACCID: - ratedCdr.CgrId = utils.FSCgrId(fieldVal) ratedCdr.AccId = fieldVal case utils.REQTYPE: ratedCdr.ReqType = fieldVal @@ -171,6 +170,7 @@ func (self *Cdrc) recordAsStoredCdr(record []string) (*utils.StoredCdr, error) { } } + ratedCdr.CgrId = utils.Sha1(ratedCdr.AccId, ratedCdr.SetupTime.String()) return ratedCdr, nil } diff --git a/cdrc/cdrc_test.go b/cdrc/cdrc_test.go index 72223f0dd..8bf6fb028 100644 --- a/cdrc/cdrc_test.go +++ b/cdrc/cdrc_test.go @@ -75,7 +75,7 @@ func TestRecordAsStoredCdr(t *testing.T) { t.Error("Failed to parse CDR in rated cdr", err) } expectedCdr := &utils.StoredCdr{ - CgrId: utils.FSCgrId(cdrRow[0]), + CgrId: utils.Sha1(cdrRow[0], time.Date(2013, 2, 3, 19, 50, 0, 0, time.UTC).String()), AccId: cdrRow[0], CdrSource: cgrConfig.CdrcSourceId, ReqType: cdrRow[1], diff --git a/cdre/csv.go b/cdre/csv.go index 7771a004d..9dfe80c52 100644 --- a/cdre/csv.go +++ b/cdre/csv.go @@ -26,16 +26,16 @@ import ( ) type CsvCdrWriter struct { - writer *csv.Writer - costShiftDigits, roundDecimals int // Round floats like Cost using this number of decimals - maskDestId string - maskLen int - exportedFields []*utils.RSRField // The fields exported, order important - firstExpOrderId, lastExpOrderId int64 + writer *csv.Writer + costShiftDigits, roundDecimals int // Round floats like Cost using this number of decimals + maskDestId string + maskLen int + exportedFields []*utils.RSRField // The fields exported, order important + firstExpOrderId, lastExpOrderId int64 } func NewCsvCdrWriter(writer io.Writer, costShiftDigits, roundDecimals int, maskDestId string, maskLen int, exportedFields []*utils.RSRField) *CsvCdrWriter { - return &CsvCdrWriter{writer:csv.NewWriter(writer), costShiftDigits: costShiftDigits, roundDecimals:roundDecimals, maskDestId:maskDestId, maskLen:maskLen, exportedFields:exportedFields} + return &CsvCdrWriter{writer: csv.NewWriter(writer), costShiftDigits: costShiftDigits, roundDecimals: roundDecimals, maskDestId: maskDestId, maskLen: maskLen, exportedFields: exportedFields} } // Return the first exported Cdr OrderId @@ -71,7 +71,6 @@ func (csvwr *CsvCdrWriter) WriteCdr(cdr *utils.StoredCdr) error { } return csvwr.writer.Write(row) - } func (csvwr *CsvCdrWriter) Close() { diff --git a/cdre/csv_test.go b/cdre/csv_test.go index 16d66163d..08b357456 100644 --- a/cdre/csv_test.go +++ b/cdre/csv_test.go @@ -32,14 +32,14 @@ func TestCsvCdrWriter(t *testing.T) { cfg, _ := config.NewDefaultCGRConfig() exportedFields := append(cfg.CdreExportedFields, &utils.RSRField{Id: "extra3"}, &utils.RSRField{Id: "dummy_extra"}, &utils.RSRField{Id: "extra1"}) csvCdrWriter := NewCsvCdrWriter(writer, 0, 4, "", -1, exportedFields) - ratedCdr := &utils.StoredCdr{CgrId: utils.FSCgrId("dsafdsaf"), AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", + ratedCdr := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(), Duration: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, ExtraFields: map[string]string{"extra1": "val_extra1", "extra2": "val_extra2", "extra3": "val_extra3"}, Cost: 1.01, } csvCdrWriter.WriteCdr(ratedCdr) csvCdrWriter.Close() - expected := `b18944ef4dc618569f24c27b9872827a242bad0c,default,dsafdsaf,192.168.1.1,rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07 08:42:25 +0000 UTC,2013-11-07 08:42:26 +0000 UTC,10,1.0100,val_extra3,"",val_extra1` + expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6,default,dsafdsaf,192.168.1.1,rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07 08:42:25 +0000 UTC,2013-11-07 08:42:26 +0000 UTC,10,1.0100,val_extra3,"",val_extra1` result := strings.TrimSpace(writer.String()) if result != expected { t.Errorf("Expected: \n%s received: \n%s.", expected, result) diff --git a/cdre/fixedwidth.go b/cdre/fixedwidth.go index a503b2bb8..36455c7b7 100644 --- a/cdre/fixedwidth.go +++ b/cdre/fixedwidth.go @@ -69,19 +69,19 @@ func NewFWCdrWriter(logDb engine.LogStorage, outFile *os.File, exportTpl *config } type FixedWidthCdrWriter struct { - logDb engine.LogStorage // Used to extract cost_details if these are requested - writer io.Writer - exportTemplate *config.CgrXmlCdreFwCfg - exportId string // Unique identifier or this export - costShiftDigits, roundDecimals int - maskDestId string - maskLen int - header, content, trailer *bytes.Buffer - firstCdrATime, lastCdrATime time.Time - numberOfRecords int - totalDuration time.Duration - totalCost float64 - firstExpOrderId, lastExpOrderId int64 + logDb engine.LogStorage // Used to extract cost_details if these are requested + writer io.Writer + exportTemplate *config.CgrXmlCdreFwCfg + exportId string // Unique identifier or this export + costShiftDigits, roundDecimals int + maskDestId string + maskLen int + header, content, trailer *bytes.Buffer + firstCdrATime, lastCdrATime time.Time + numberOfRecords int + totalDuration time.Duration + totalCost float64 + firstExpOrderId, lastExpOrderId int64 } // Return Json marshaled callCost attached to diff --git a/cdre/fixedwidth_test.go b/cdre/fixedwidth_test.go index 0e17bb395..6dc3a7d92 100644 --- a/cdre/fixedwidth_test.go +++ b/cdre/fixedwidth_test.go @@ -81,7 +81,7 @@ func TestWriteCdr(t *testing.T) { Trailer: &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds}, } fwWriter := FixedWidthCdrWriter{writer: wrBuf, exportTemplate: exportTpl, roundDecimals: 4, header: &bytes.Buffer{}, content: &bytes.Buffer{}, trailer: &bytes.Buffer{}} - cdr := &utils.StoredCdr{CgrId: utils.FSCgrId("dsafdsaf"), OrderId: 1, AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", + cdr := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 1, AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), @@ -139,14 +139,14 @@ func TestWriteCdrs(t *testing.T) { Trailer: &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds}, } fwWriter := FixedWidthCdrWriter{writer: wrBuf, exportTemplate: exportTpl, roundDecimals: 4, header: &bytes.Buffer{}, content: &bytes.Buffer{}, trailer: &bytes.Buffer{}} - cdr1 := &utils.StoredCdr{CgrId: utils.FSCgrId("aaa1"), OrderId: 2, AccId: "aaa1", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", + cdr1 := &utils.StoredCdr{CgrId: utils.Sha1("aaa1", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), OrderId: 2, AccId: "aaa1", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1001", Subject: "1001", Destination: "1010", SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.25, ExtraFields: map[string]string{"productnumber": "12341", "fieldextr2": "valextr2"}, } - cdr2 := &utils.StoredCdr{CgrId: utils.FSCgrId("aaa2"), OrderId: 4, AccId: "aaa2", CdrHost: "192.168.1.2", ReqType: "prepaid", Direction: "*out", Tenant: "cgrates.org", + cdr2 := &utils.StoredCdr{CgrId: utils.Sha1("aaa2", time.Date(2013, 11, 7, 7, 42, 20, 0, time.UTC).String()), OrderId: 4, AccId: "aaa2", CdrHost: "192.168.1.2", ReqType: "prepaid", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1002", Subject: "1002", Destination: "1011", SetupTime: time.Date(2013, 11, 7, 7, 42, 20, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 7, 42, 26, 0, time.UTC), @@ -154,7 +154,7 @@ func TestWriteCdrs(t *testing.T) { ExtraFields: map[string]string{"productnumber": "12342", "fieldextr2": "valextr2"}, } cdr3 := &utils.StoredCdr{} - cdr4 := &utils.StoredCdr{CgrId: utils.FSCgrId("aaa3"), OrderId: 3, AccId: "aaa4", CdrHost: "192.168.1.4", ReqType: "postpaid", Direction: "*out", Tenant: "cgrates.org", + cdr4 := &utils.StoredCdr{CgrId: utils.Sha1("aaa3", time.Date(2013, 11, 7, 9, 42, 18, 0, time.UTC).String()), OrderId: 3, AccId: "aaa4", CdrHost: "192.168.1.4", ReqType: "postpaid", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1004", Subject: "1004", Destination: "1013", SetupTime: time.Date(2013, 11, 7, 9, 42, 18, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 9, 42, 26, 0, time.UTC), diff --git a/cdrs/fscdr.go b/cdrs/fscdr.go index 65f09f9dd..769952c06 100644 --- a/cdrs/fscdr.go +++ b/cdrs/fscdr.go @@ -76,7 +76,8 @@ func (fsCdr FSCdr) New(body []byte) (utils.RawCDR, error) { } func (fsCdr FSCdr) GetCgrId() string { - return utils.FSCgrId(fsCdr.vars[FS_UUID]) + setupTime, _ := utils.ParseTimeDetectLayout(fsCdr.vars[FS_SETUP_TIME]) + return utils.Sha1(fsCdr.vars[FS_UUID], setupTime.String()) } func (fsCdr FSCdr) GetAccId() string { return fsCdr.vars[FS_UUID] @@ -154,13 +155,11 @@ func (fsCdr FSCdr) searchExtraField(field string, body map[string]interface{}) ( } func (fsCdr FSCdr) GetSetupTime() (t time.Time, err error) { - //ToDo: Make sure we work with UTC instead of local time at, err := strconv.ParseInt(fsCdr.vars[FS_SETUP_TIME], 0, 64) t = time.Unix(at, 0) return } func (fsCdr FSCdr) GetAnswerTime() (t time.Time, err error) { - //ToDo: Make sure we work with UTC instead of local time at, err := strconv.ParseInt(fsCdr.vars[FS_ANSWER_TIME], 0, 64) t = time.Unix(at, 0) return @@ -224,8 +223,6 @@ func (fsCdr FSCdr) AsStoredCdr(runId, reqTypeFld, directionFld, tenantFld, torFl } else { // Not mandatory, need to generate here CgrId rtCdr.CgrId = utils.GenUUID() } - } else { // hasKey, use it to generate cgrid - rtCdr.CgrId = utils.FSCgrId(rtCdr.AccId) } if rtCdr.CdrHost = fsCdr.GetCdrHost(); len(rtCdr.CdrHost) == 0 && fieldsMandatory { return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, utils.CDRHOST)) @@ -298,6 +295,7 @@ func (fsCdr FSCdr) AsStoredCdr(runId, reqTypeFld, directionFld, tenantFld, torFl return nil, err } } + rtCdr.CgrId = utils.Sha1(rtCdr.AccId, rtCdr.SetupTime.String()) rtCdr.ExtraFields = make(map[string]string, len(extraFlds)) for _, fldName := range extraFlds { if fldVal, hasKey := fsCdr.vars[fldName]; !hasKey && fieldsMandatory { diff --git a/cdrs/fscdr_test.go b/cdrs/fscdr_test.go index 0bdc6f12f..b640d97ba 100644 --- a/cdrs/fscdr_test.go +++ b/cdrs/fscdr_test.go @@ -47,7 +47,8 @@ func TestCDRFields(t *testing.T) { if err != nil { t.Errorf("Error loading cdr: %v", err) } - if fsCdr.GetCgrId() != utils.FSCgrId("01df56f4-d99a-4ef6-b7fe-b924b2415b7f") { + setupTime, _ := fsCdr.GetSetupTime() + if fsCdr.GetCgrId() != utils.Sha1("01df56f4-d99a-4ef6-b7fe-b924b2415b7f", setupTime.String()) { t.Error("Error parsing cdr: ", fsCdr) } if fsCdr.GetAccId() != "01df56f4-d99a-4ef6-b7fe-b924b2415b7f" { @@ -77,7 +78,6 @@ func TestCDRFields(t *testing.T) { if fsCdr.GetReqType() != utils.RATED { t.Error("Error parsing cdr: ", fsCdr) } - setupTime, _ := fsCdr.GetSetupTime() expectedSTime, _ := time.Parse(time.RFC3339, "2013-08-04T09:50:54Z") if setupTime.UTC() != expectedSTime { t.Error("Error parsing cdr: ", fsCdr) @@ -112,7 +112,7 @@ func TestFsCdrAsStoredCdr(t *testing.T) { if err != nil { t.Error("Unexpected error received", err) } - expctRatedCdr := &utils.StoredCdr{CgrId: utils.FSCgrId("01df56f4-d99a-4ef6-b7fe-b924b2415b7f"), AccId: "01df56f4-d99a-4ef6-b7fe-b924b2415b7f", + expctRatedCdr := &utils.StoredCdr{CgrId: utils.Sha1("01df56f4-d99a-4ef6-b7fe-b924b2415b7f", time.Date(2013, 8, 4, 9, 50, 54, 0, time.UTC).Local().String()), AccId: "01df56f4-d99a-4ef6-b7fe-b924b2415b7f", CdrHost: "127.0.0.1", CdrSource: FS_CDR_SOURCE, ReqType: utils.RATED, Direction: "*out", Tenant: "ipbx.itsyscom.com", TOR: "call", Account: "dan", Subject: "dan", Destination: "+4986517174963", SetupTime: time.Date(2013, 8, 4, 9, 50, 54, 0, time.UTC).Local(), @@ -126,7 +126,7 @@ func TestFsCdrAsStoredCdr(t *testing.T) { if err != nil { t.Error("Unexpected error received", err) } - expctRatedCdr2 := &utils.StoredCdr{CgrId: utils.FSCgrId("01df56f4-d99a-4ef6-b7fe-b924b2415b7f"), AccId: "01df56f4-d99a-4ef6-b7fe-b924b2415b7f", CdrHost: "127.0.0.1", + expctRatedCdr2 := &utils.StoredCdr{CgrId: utils.Sha1("01df56f4-d99a-4ef6-b7fe-b924b2415b7f", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()), AccId: "01df56f4-d99a-4ef6-b7fe-b924b2415b7f", CdrHost: "127.0.0.1", CdrSource: FS_CDR_SOURCE, ReqType: "postpaid", Direction: "*in", Tenant: "cgrates.com", TOR: "premium_call", Account: "first_account", Subject: "first_subject", Destination: "+4986517174963", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), diff --git a/charging_tests/ddazmbl1_test.go b/charging_tests/ddazmbl1_test.go index 9f9669ab5..85c13696c 100644 --- a/charging_tests/ddazmbl1_test.go +++ b/charging_tests/ddazmbl1_test.go @@ -115,7 +115,7 @@ TOPUP10_AT,TOPUP10_AC1,ASAP,10` func TestExecuteActions(t *testing.T) { scheduler.NewScheduler().LoadActionTimings(acntDb) - time.Sleep(time.Duration(1) * time.Microsecond) // Give time to scheduler to topup the account + time.Sleep(time.Duration(500) * time.Microsecond) // Give time to scheduler to topup the account if acnt, err := acntDb.GetAccount("*out:cgrates.org:12345"); err != nil { t.Error(err) } else if len(acnt.BalanceMap) != 2 { diff --git a/engine/action_timing.go b/engine/action_timing.go index 0ba9ea304..552432e7f 100644 --- a/engine/action_timing.go +++ b/engine/action_timing.go @@ -236,9 +236,9 @@ func (at *ActionTiming) Execute() (err error) { _, err := AccLock.Guard(ubId, func() (float64, error) { ub, err := accountingStorage.GetAccount(ubId) if err != nil { - Logger.Warning(fmt.Sprintf("Could not get user balances for this id: %s. Skipping!", ubId)) - return 0, err - } else if ub.Disabled { + Logger.Warning(fmt.Sprintf("Could not get user balances for this id: %s. Skipping!", ubId)) + return 0, err + } else if ub.Disabled { return 0, fmt.Errorf("User %s is disabled", ubId) } if err != nil { diff --git a/engine/rateinterval.go b/engine/rateinterval.go index a7522c1a6..eecd708d1 100644 --- a/engine/rateinterval.go +++ b/engine/rateinterval.go @@ -48,7 +48,7 @@ type RITiming struct { } func (rit *RITiming) Stringify() string { - return utils.SHA1(fmt.Sprintf("%v", rit))[:8] + return utils.Sha1(fmt.Sprintf("%v", rit))[:8] } // Separate structure used for rating plan size optimization @@ -64,7 +64,7 @@ func (rir *RIRate) Stringify() string { for _, r := range rir.Rates { str += r.Stringify() } - return utils.SHA1(str)[:8] + return utils.Sha1(str)[:8] } type Rate struct { @@ -75,7 +75,7 @@ type Rate struct { } func (r *Rate) Stringify() string { - return utils.SHA1(fmt.Sprintf("%v", r))[:8] + return utils.Sha1(fmt.Sprintf("%v", r))[:8] } func (p *Rate) Equal(o *Rate) bool { diff --git a/engine/storage_interface.go b/engine/storage_interface.go index d8dc334a4..a5852ae26 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -114,11 +114,11 @@ type CdrStorage interface { type LogStorage interface { Storage //GetAllActionTimingsLogs() (map[string]ActionsTimings, error) - LogCallCost(uuid, source, runid string, cc *CallCost) error + LogCallCost(cgrid, source, runid string, cc *CallCost) error LogError(uuid, source, runid, errstr string) error LogActionTrigger(ubId, source string, at *ActionTrigger, as Actions) error LogActionTiming(source string, at *ActionTiming, as Actions) error - GetCallCostLog(uuid, source, runid string) (*CallCost, error) + GetCallCostLog(cgrid, source, runid string) (*CallCost, error) } type LoadStorage interface { diff --git a/engine/storage_map.go b/engine/storage_map.go index fb0a86303..517e698c4 100644 --- a/engine/storage_map.go +++ b/engine/storage_map.go @@ -428,14 +428,14 @@ func (ms *MapStorage) GetAllActionTimings() (ats map[string]ActionPlan, err erro return } -func (ms *MapStorage) LogCallCost(uuid, source, runid string, cc *CallCost) error { +func (ms *MapStorage) LogCallCost(cgrid, source, runid string, cc *CallCost) error { result, err := ms.ms.Marshal(cc) - ms.dict[LOG_CALL_COST_PREFIX+source+runid+"_"+uuid] = result + ms.dict[LOG_CALL_COST_PREFIX+source+runid+"_"+cgrid] = result return err } -func (ms *MapStorage) GetCallCostLog(uuid, source, runid string) (cc *CallCost, err error) { - if values, ok := ms.dict[LOG_CALL_COST_PREFIX+source+runid+"_"+uuid]; ok { +func (ms *MapStorage) GetCallCostLog(cgrid, source, runid string) (cc *CallCost, err error) { + if values, ok := ms.dict[LOG_CALL_COST_PREFIX+source+runid+"_"+cgrid]; ok { err = ms.ms.Unmarshal(values, &cc) } else { return nil, errors.New("not found") diff --git a/engine/storage_mongo.go b/engine/storage_mongo.go index 92b387f2b..c546fc734 100644 --- a/engine/storage_mongo.go +++ b/engine/storage_mongo.go @@ -213,13 +213,13 @@ func (ms *MongoStorage) GetAllActionTimings() (ats map[string]ActionPlan, err er return } -func (ms *MongoStorage) LogCallCost(uuid, source string, cc *CallCost) error { - return ms.db.C("cclog").Insert(&LogCostEntry{uuid, cc, source}) +func (ms *MongoStorage) LogCallCost(cgrid, source string, cc *CallCost) error { + return ms.db.C("cclog").Insert(&LogCostEntry{cgrid, cc, source}) } -func (ms *MongoStorage) GetCallCostLog(uuid, source string) (cc *CallCost, err error) { +func (ms *MongoStorage) GetCallCostLog(cgrid, source string) (cc *CallCost, err error) { result := new(LogCostEntry) - err = ms.db.C("cclog").Find(bson.M{"_id": uuid, "source": source}).One(result) + err = ms.db.C("cclog").Find(bson.M{"_id": cgrid, "source": source}).One(result) cc = result.CallCost return } diff --git a/engine/storage_redis.go b/engine/storage_redis.go index 82dd1ddc5..2dde015ab 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -524,19 +524,19 @@ func (rs *RedisStorage) GetAllActionTimings() (ats map[string]ActionPlan, err er return } -func (rs *RedisStorage) LogCallCost(uuid, source, runid string, cc *CallCost) (err error) { +func (rs *RedisStorage) LogCallCost(cgrid, source, runid string, cc *CallCost) (err error) { var result []byte result, err = rs.ms.Marshal(cc) if err != nil { return } - err = rs.db.Set(LOG_CALL_COST_PREFIX+source+runid+"_"+uuid, result) + err = rs.db.Set(LOG_CALL_COST_PREFIX+source+runid+"_"+cgrid, result) return } -func (rs *RedisStorage) GetCallCostLog(uuid, source, runid string) (cc *CallCost, err error) { +func (rs *RedisStorage) GetCallCostLog(cgrid, source, runid string) (cc *CallCost, err error) { var values []byte - if values, err = rs.db.Get(LOG_CALL_COST_PREFIX + source + runid + "_" + uuid); err == nil { + if values, err = rs.db.Get(LOG_CALL_COST_PREFIX + source + runid + "_" + cgrid); err == nil { err = rs.ms.Unmarshal(values, cc) } return diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 424fe03dd..51e44ab95 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -461,7 +461,7 @@ func (self *SQLStorage) SetTPAccountActions(tpid string, aa map[string]*utils.TP return nil } -func (self *SQLStorage) LogCallCost(uuid, source, runid string, cc *CallCost) (err error) { +func (self *SQLStorage) LogCallCost(cgrid, source, runid string, cc *CallCost) (err error) { //ToDo: Add cgrid to logCallCost if self.Db == nil { //timespans.Logger.Warning("Cannot write log to database.") @@ -471,10 +471,9 @@ func (self *SQLStorage) LogCallCost(uuid, source, runid string, cc *CallCost) (e if err != nil { Logger.Err(fmt.Sprintf("Error marshalling timespans to json: %v", err)) } - _, err = self.Db.Exec(fmt.Sprintf("INSERT INTO %s (cgrid, accid, direction, tenant, tor, account, subject, destination, cost, timespans, source, runid, cost_time)VALUES ('%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', %f, '%s','%s','%s',now()) ON DUPLICATE KEY UPDATE direction=values(direction), tenant=values(tenant), tor=values(tor), account=values(account), subject=values(subject), destination=values(destination), cost=values(cost), timespans=values(timespans), source=values(source), cost_time=now()", + _, err = self.Db.Exec(fmt.Sprintf("INSERT INTO %s (cgrid, direction, tenant, tor, account, subject, destination, cost, timespans, source, runid, cost_time)VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', %f, '%s','%s','%s',now()) ON DUPLICATE KEY UPDATE direction=values(direction), tenant=values(tenant), tor=values(tor), account=values(account), subject=values(subject), destination=values(destination), cost=values(cost), timespans=values(timespans), source=values(source), cost_time=now()", utils.TBL_COST_DETAILS, - utils.FSCgrId(uuid), - uuid, + cgrid, cc.Direction, cc.Tenant, cc.TOR, @@ -492,16 +491,16 @@ func (self *SQLStorage) LogCallCost(uuid, source, runid string, cc *CallCost) (e } func (self *SQLStorage) GetCallCostLog(cgrid, source, runid string) (cc *CallCost, err error) { - qry := fmt.Sprintf("SELECT cgrid, accid, direction, tenant, tor, account, subject, destination, cost, timespans, source FROM %s WHERE cgrid='%s' AND runid='%s'", + qry := fmt.Sprintf("SELECT cgrid, direction, tenant, tor, account, subject, destination, cost, timespans, source FROM %s WHERE cgrid='%s' AND runid='%s'", utils.TBL_COST_DETAILS, cgrid, runid) if len(source) != 0 { qry += fmt.Sprintf(" AND source='%s'", source) } row := self.Db.QueryRow(qry) - var accid, src string + var src string var timespansJson string cc = &CallCost{Cost: -1} - err = row.Scan(&cgrid, &accid, &cc.Direction, &cc.Tenant, &cc.TOR, &cc.Account, &cc.Subject, + err = row.Scan(&cgrid, &cc.Direction, &cc.Tenant, &cc.TOR, &cc.Account, &cc.Subject, &cc.Destination, &cc.Cost, ×pansJson, &src) if len(timespansJson) == 0 { // No costs returned return nil, nil diff --git a/engine/storage_sql_local_test.go b/engine/storage_sql_local_test.go index 17ad04323..eca87edf6 100644 --- a/engine/storage_sql_local_test.go +++ b/engine/storage_sql_local_test.go @@ -157,21 +157,24 @@ func TestSetCdr(t *testing.T) { t.Error(err.Error()) } } - strCdr1 := &utils.StoredCdr{CgrId: utils.FSCgrId("bbb1"), AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: "rated", + strCdr1 := &utils.StoredCdr{AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201} - strCdr2 := &utils.StoredCdr{CgrId: utils.FSCgrId("bbb2"), AccId: "bbb2", CdrHost: "192.168.1.2", CdrSource: "UNKNOWN2", ReqType: "prepaid", + strCdr1.CgrId = utils.Sha1(strCdr1.AccId, strCdr1.SetupTime.String()) + strCdr2 := &utils.StoredCdr{AccId: "bbb2", CdrHost: "192.168.1.2", CdrSource: "UNKNOWN2", ReqType: "prepaid", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(12) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: utils.DEFAULT_RUNID, Cost: 0.201} - strCdr3 := &utils.StoredCdr{CgrId: utils.FSCgrId("bbb3"), AccId: "bbb3", CdrHost: "192.168.1.1", CdrSource: TEST_SQL, ReqType: "rated", + strCdr2.CgrId = utils.Sha1(strCdr2.AccId, strCdr2.SetupTime.String()) + strCdr3 := &utils.StoredCdr{AccId: "bbb3", CdrHost: "192.168.1.1", CdrSource: TEST_SQL, ReqType: "rated", Direction: "*out", Tenant: "itsyscom.com", TOR: "call", Account: "1002", Subject: "1000", Destination: "+4986517174963", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201} + strCdr3.CgrId = utils.Sha1(strCdr3.AccId, strCdr3.SetupTime.String()) for _, cdr := range []*utils.StoredCdr{strCdr1, strCdr2, strCdr3} { if err := mysql.SetCdr(cdr); err != nil { @@ -184,21 +187,24 @@ func TestSetRatedCdr(t *testing.T) { if !*testLocal { return } - strCdr1 := &utils.StoredCdr{CgrId: utils.FSCgrId("bbb1"), AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: "rated", + strCdr1 := &utils.StoredCdr{AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201} - strCdr2 := &utils.StoredCdr{CgrId: utils.FSCgrId("bbb2"), AccId: "bbb2", CdrHost: "192.168.1.2", CdrSource: "UNKNOWN", ReqType: "prepaid", + strCdr1.CgrId = utils.Sha1(strCdr1.AccId, strCdr1.SetupTime.String()) + strCdr2 := &utils.StoredCdr{AccId: "bbb2", CdrHost: "192.168.1.2", CdrSource: "UNKNOWN", ReqType: "prepaid", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(12) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: utils.DEFAULT_RUNID, Cost: 0.201} - strCdr3 := &utils.StoredCdr{CgrId: utils.FSCgrId("bbb3"), AccId: "bbb3", CdrHost: "192.168.1.1", CdrSource: TEST_SQL, ReqType: "rated", + strCdr2.CgrId = utils.Sha1(strCdr2.AccId, strCdr2.SetupTime.String()) + strCdr3 := &utils.StoredCdr{AccId: "bbb3", CdrHost: "192.168.1.1", CdrSource: TEST_SQL, ReqType: "rated", Direction: "*out", Tenant: "itsyscom.com", TOR: "call", Account: "1002", Subject: "1002", Destination: "+4986517174964", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: "wholesale_run", Cost: 1.201} + strCdr3.CgrId = utils.Sha1(strCdr3.AccId, strCdr3.SetupTime.String()) for _, cdr := range []*utils.StoredCdr{strCdr1, strCdr2, strCdr3} { if err := mysql.SetRatedCdr(cdr, ""); err != nil { @@ -219,14 +225,16 @@ func TestGetStoredCdrs(t *testing.T) { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Filter on cgrids - if storedCdrs, err := mysql.GetStoredCdrs([]string{utils.FSCgrId("bbb1"), utils.FSCgrId("bbb2")}, + if storedCdrs, err := mysql.GetStoredCdrs([]string{utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()), + utils.Sha1("bbb2", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String())}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 0, 0, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 2 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Filter on cgrids plus reqType - if storedCdrs, err := mysql.GetStoredCdrs([]string{utils.FSCgrId("bbb1"), utils.FSCgrId("bbb2")}, + if storedCdrs, err := mysql.GetStoredCdrs([]string{utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()), + utils.Sha1("bbb2", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String())}, nil, nil, nil, []string{"prepaid"}, nil, nil, nil, nil, nil, nil, 0, 0, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 1 { @@ -341,7 +349,7 @@ func TestGetStoredCdrs(t *testing.T) { if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, []string{"1001", "+498651"}, 0, 0, timeStart, timeEnd, false, false); err != nil { t.Error(err.Error()) } else if len(storedCdrs) != 4 { - t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) + t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } // Filter on ignoreErr if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 0, 0, timeStart, timeEnd, true, false); err != nil { @@ -403,7 +411,7 @@ func TestCallCost(t *testing.T) { if !*testLocal { return } - cgrId := utils.FSCgrId("bbb1") + cgrId := utils.Sha1("bbb1", "123") cc := &CallCost{ Timespans: []*TimeSpan{ &TimeSpan{ @@ -416,7 +424,7 @@ func TestCallCost(t *testing.T) { }, }, } - if err := mysql.LogCallCost("bbb1", TEST_SQL, TEST_SQL, cc); err != nil { + if err := mysql.LogCallCost(cgrId, TEST_SQL, TEST_SQL, cc); err != nil { t.Error(err.Error()) } if ccRcv, err := mysql.GetCallCostLog(cgrId, TEST_SQL, TEST_SQL); err != nil { @@ -431,7 +439,8 @@ func TestRemStoredCdrs(t *testing.T) { return } var timeStart, timeEnd time.Time - if err := mysql.RemStoredCdrs([]string{utils.FSCgrId("bbb1")}); err != nil { + cgrIdB1 := utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()) + if err := mysql.RemStoredCdrs([]string{cgrIdB1}); err != nil { t.Error(err.Error()) } if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 0, 0, timeStart, timeEnd, false, false); err != nil { @@ -439,8 +448,20 @@ func TestRemStoredCdrs(t *testing.T) { } else if len(storedCdrs) != 7 { t.Error("Unexpected number of StoredCdrs returned: ", storedCdrs) } - if err := mysql.RemStoredCdrs([]string{utils.FSCgrId("aaa1"), utils.FSCgrId("aaa2"), utils.FSCgrId("aaa3"), utils.FSCgrId("aaa4"), utils.FSCgrId("aaa5"), - utils.FSCgrId("bbb2"), utils.FSCgrId("bbb3")}); err != nil { + tm, _ := utils.ParseTimeDetectLayout("2013-11-08T08:42:20Z") + cgrIdA1 := utils.Sha1("aaa1", tm.String()) + tm, _ = utils.ParseTimeDetectLayout("2013-11-08T08:42:22Z") + cgrIdA2 := utils.Sha1("aaa2", tm.String()) + tm, _ = utils.ParseTimeDetectLayout("2013-11-07T08:42:24Z") + cgrIdA3 := utils.Sha1("aaa3", tm.String()) + tm, _ = utils.ParseTimeDetectLayout("2013-11-07T08:42:21Z") + cgrIdA4 := utils.Sha1("aaa4", tm.String()) + tm, _ = utils.ParseTimeDetectLayout("2013-11-07T08:42:25Z") + cgrIdA5 := utils.Sha1("aaa5", tm.String()) + cgrIdB2 := utils.Sha1("bbb2", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()) + cgrIdB3 := utils.Sha1("bbb3", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()) + if err := mysql.RemStoredCdrs([]string{cgrIdA1, cgrIdA2, cgrIdA3, cgrIdA4, cgrIdA5, + cgrIdB2, cgrIdB3}); err != nil { t.Error(err.Error()) } if storedCdrs, err := mysql.GetStoredCdrs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 0, 0, timeStart, timeEnd, false, false); err != nil { diff --git a/mediator/fsfilecsvcdr.go b/mediator/fsfilecsvcdr.go index 134a0b07b..be3e004b3 100644 --- a/mediator/fsfilecsvcdr.go +++ b/mediator/fsfilecsvcdr.go @@ -35,20 +35,21 @@ type FScsvCDR struct { torIdx, accountIdx, destinationIdx, + setupTimeIdx, answerTimeIdx, durationIdx int // Field indexes cgrCfg *config.CGRConfig // CGR Config instance } func NewFScsvCDR(cdrRow []string, accIdIdx, subjectIdx, reqtypeIdx, directionIdx, tenantIdx, torIdx, - accountIdx, destinationIdx, answerTimeIdx, durationIdx int, cfg *config.CGRConfig) (*FScsvCDR, error) { + accountIdx, destinationIdx, setupTimeIdx, answerTimeIdx, durationIdx int, cfg *config.CGRConfig) (*FScsvCDR, error) { fscdr := FScsvCDR{cdrRow, accIdIdx, subjectIdx, reqtypeIdx, directionIdx, tenantIdx, - torIdx, accountIdx, destinationIdx, answerTimeIdx, durationIdx, cfg} + torIdx, accountIdx, destinationIdx, setupTimeIdx, answerTimeIdx, durationIdx, cfg} return &fscdr, nil } func (self *FScsvCDR) GetCgrId() string { - return utils.FSCgrId(self.rowData[self.accIdIdx]) + return utils.Sha1(self.rowData[self.accIdIdx], self.rowData[self.setupTimeIdx]) } func (self *FScsvCDR) GetAccId() string { diff --git a/mediator/mediator.go b/mediator/mediator.go index 3a0835f20..bda8c9589 100644 --- a/mediator/mediator.go +++ b/mediator/mediator.go @@ -105,7 +105,7 @@ func (self *Mediator) getCostsFromRater(cdr *utils.StoredCdr) (*engine.CallCost, self.logDb.LogError(cdr.CgrId, engine.MEDIATOR_SOURCE, cdr.MediationRunId, err.Error()) } else { // If the mediator calculated a price it will write it to logdb - self.logDb.LogCallCost(cdr.AccId, engine.MEDIATOR_SOURCE, cdr.MediationRunId, cc) + self.logDb.LogCallCost(utils.Sha1(cdr.AccId, cdr.SetupTime.String()), engine.MEDIATOR_SOURCE, cdr.MediationRunId, cc) } return cc, err } diff --git a/mediator/mediator_rpc.go b/mediator/mediator_rpc.go index 0fcc7fd62..8cffe0ce0 100644 --- a/mediator/mediator_rpc.go +++ b/mediator/mediator_rpc.go @@ -20,8 +20,8 @@ package mediator import ( "fmt" - "time" "github.com/cgrates/cgrates/utils" + "time" ) type MediatorV1 struct { diff --git a/sessionmanager/event.go b/sessionmanager/event.go index 20df15e5c..721dfd56d 100644 --- a/sessionmanager/event.go +++ b/sessionmanager/event.go @@ -25,6 +25,7 @@ import ( type Event interface { New(string) Event GetName() string + GetCgrId() string GetUUID() string GetDirection(string) string GetSubject(string) string diff --git a/sessionmanager/fsevent.go b/sessionmanager/fsevent.go index adfc2afe1..32f780693 100644 --- a/sessionmanager/fsevent.go +++ b/sessionmanager/fsevent.go @@ -121,6 +121,9 @@ func (fsev FSEvent) GetTOR(fieldName string) string { } return utils.FirstNonEmpty(fsev[fieldName], fsev[TOR], config.CgrConfig().DefaultTOR) } +func (fsev FSEvent) GetCgrId() string { + return utils.Sha1(fsev[UUID], fsev[SETUP_TIME]) +} func (fsev FSEvent) GetUUID() string { return fsev[UUID] } diff --git a/sessionmanager/session.go b/sessionmanager/session.go index 532d2306e..6e4d1cb7d 100644 --- a/sessionmanager/session.go +++ b/sessionmanager/session.go @@ -30,6 +30,7 @@ import ( // Session type holding the call information fields, a session delegate for specific // actions and a channel to signal end of the debit loop. type Session struct { + cgrid string uuid string stopDebit chan bool sessionManager SessionManager @@ -45,7 +46,8 @@ type SessionRun struct { // Creates a new session and in case of prepaid starts the debit loop for each of the session runs individually func NewSession(ev Event, sm SessionManager) *Session { - s := &Session{uuid: ev.GetUUID(), + s := &Session{cgrid: ev.GetCgrId(), + uuid: ev.GetUUID(), stopDebit: make(chan bool), sessionManager: sm, sessionRuns: make([]*SessionRun, 0), @@ -169,7 +171,7 @@ func (s *Session) SaveOperations() { if s.sessionManager.GetDbLogger() == nil { engine.Logger.Err(" Error: no connection to logger database, cannot save costs") } - s.sessionManager.GetDbLogger().LogCallCost(s.uuid, engine.SESSION_MANAGER_SOURCE, sr.runId, firstCC) + s.sessionManager.GetDbLogger().LogCallCost(s.cgrid, engine.SESSION_MANAGER_SOURCE, sr.runId, firstCC) } }() } diff --git a/utils/apitpdata.go b/utils/apitpdata.go index 4c41e923c..d3dc861cb 100644 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -335,8 +335,8 @@ type AttrExpFileCdrs struct { Account []string // If provided, it will filter account Subject []string // If provided, it will filter the rating subject DestinationPrefix []string // If provided, it will filter on destination prefix - OrderIdStart int64 // Export from this order identifier - OrderIdEnd int64 // Export smaller than this order identifier + OrderIdStart int64 // Export from this order identifier + OrderIdEnd int64 // Export smaller than this order identifier TimeStart string // If provided, it will represent the starting of the CDRs interval (>=) TimeEnd string // If provided, it will represent the end of the CDRs interval (<) SkipErrors bool // Do not export errored CDRs @@ -344,12 +344,12 @@ type AttrExpFileCdrs struct { } type ExportedFileCdrs struct { - ExportedFilePath string // Full path to the newly generated export file - TotalRecords int // Number of CDRs to be exported - FirstOrderId, LastOrderId int64 // The order id of the last exported CDR - ExportedCgrIds []string // List of successfuly exported cgrids in the file - UnexportedCgrIds map[string]string // Map of errored CDRs, map key is cgrid, value will be the error string - + ExportedFilePath string // Full path to the newly generated export file + TotalRecords int // Number of CDRs to be exported + FirstOrderId, LastOrderId int64 // The order id of the last exported CDR + ExportedCgrIds []string // List of successfuly exported cgrids in the file + UnexportedCgrIds map[string]string // Map of errored CDRs, map key is cgrid, value will be the error string + } type AttrRemCdrs struct { diff --git a/utils/cgrcdr.go b/utils/cgrcdr.go index d5147d754..15b11b560 100644 --- a/utils/cgrcdr.go +++ b/utils/cgrcdr.go @@ -43,7 +43,8 @@ func NewCgrCdrFromHttpReq(req *http.Request) (CgrCdr, error) { type CgrCdr map[string]string func (cgrCdr CgrCdr) GetCgrId() string { - return FSCgrId(cgrCdr[ACCID]) + setupTime, _ := cgrCdr.GetSetupTime() + return Sha1(cgrCdr[ACCID], setupTime.String()) } func (cgrCdr CgrCdr) GetAccId() string { @@ -123,11 +124,7 @@ func (cgrCdr CgrCdr) AsStoredCdr(runId, reqTypeFld, directionFld, tenantFld, tor if rtCdr.AccId, hasKey = cgrCdr[ACCID]; !hasKey { if fieldsMandatory { return nil, errors.New(fmt.Sprintf("%s:%s", ERR_MANDATORY_IE_MISSING, ACCID)) - } else { // Not mandatory, cgrid needs however to be unique - rtCdr.CgrId = GenUUID() } - } else { // hasKey, use it to generate cgrid - rtCdr.CgrId = FSCgrId(rtCdr.AccId) } if rtCdr.CdrHost, hasKey = cgrCdr[CDRHOST]; !hasKey && fieldsMandatory { return nil, errors.New(fmt.Sprintf("%s:%s", ERR_MANDATORY_IE_MISSING, CDRHOST)) @@ -208,5 +205,6 @@ func (cgrCdr CgrCdr) AsStoredCdr(runId, reqTypeFld, directionFld, tenantFld, tor rtCdr.ExtraFields[fldName] = fldVal } } + rtCdr.CgrId = Sha1(rtCdr.AccId, rtCdr.SetupTime.String()) return rtCdr, nil } diff --git a/utils/cgrcdr_test.go b/utils/cgrcdr_test.go index 2e1c48d87..f9428052b 100644 --- a/utils/cgrcdr_test.go +++ b/utils/cgrcdr_test.go @@ -30,9 +30,10 @@ curl --data "accid=asbfdsaf&cdrhost=192.168.1.1&reqtype=rated&direction=*out&ten func TestCgrCdrFields(t *testing.T) { cgrCdr := CgrCdr{"accid": "dsafdsaf", "cdrhost": "192.168.1.1", "reqtype": "rated", "direction": "*out", "tenant": "cgrates.org", "tor": "call", - "account": "1001", "subject": "1001", "destination": "1002", "answer_time": "2013-11-07T08:42:26Z", "duration": "10", + "account": "1001", "subject": "1001", "destination": "1002", "setup_time": "2013-11-07T08:42:20Z", "answer_time": "2013-11-07T08:42:26Z", "duration": "10", "field_extr1": "val_extr1", "fieldextr2": "valextr2"} - if cgrCdr.GetCgrId() != FSCgrId("dsafdsaf") { + setupTime, _ := ParseTimeDetectLayout("2013-11-07T08:42:20Z") + if cgrCdr.GetCgrId() != Sha1("dsafdsaf", setupTime.String()) { t.Error("Error parsing cdr: ", cgrCdr) } if cgrCdr.GetAccId() != "dsafdsaf" { @@ -62,6 +63,10 @@ func TestCgrCdrFields(t *testing.T) { if cgrCdr.GetReqType() != RATED { t.Error("Error parsing cdr: ", cgrCdr) } + expectedSTime, _ := time.Parse(time.RFC3339, "2013-11-07T08:42:20Z") + if setupTime.UTC() != expectedSTime { + t.Error("Error parsing cdr: ", cgrCdr) + } answerTime, _ := cgrCdr.GetAnswerTime() expectedATime, _ := time.Parse(time.RFC3339, "2013-11-07T08:42:26Z") if answerTime.UTC() != expectedATime { @@ -88,7 +93,8 @@ func TestCgrCdrAsStoredCdr(t *testing.T) { if err != nil { t.Error("Unexpected error received", err) } - expctRatedCdr := &StoredCdr{CgrId: FSCgrId("dsafdsaf"), AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "source_test", ReqType: "rated", + setupTime, _ := ParseTimeDetectLayout("2013-11-07T08:42:24Z") + expctRatedCdr := &StoredCdr{CgrId: Sha1("dsafdsaf", setupTime.String()), AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "source_test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813744, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(), Duration: 10000000000, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: "wholesale_run", Cost: -1} @@ -100,7 +106,7 @@ func TestCgrCdrAsStoredCdr(t *testing.T) { if err != nil { t.Error("Unexpected error received", err) } - expctRatedCdr2 := &StoredCdr{CgrId: FSCgrId("dsafdsaf"), AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "source_test", ReqType: "postpaid", + expctRatedCdr2 := &StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()), AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "source_test", ReqType: "postpaid", Direction: "*in", Tenant: "cgrates.com", TOR: "premium_call", Account: "first_account", Subject: "first_subject", Destination: "1002", SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(12) * time.Second, diff --git a/utils/coreutils.go b/utils/coreutils.go index a4c78c212..ef49791bc 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -41,18 +41,16 @@ func FirstNonEmpty(vals ...string) string { return "" } -func SHA1(text string) string { +func Sha1(attrs ...string) string { hasher := sha1.New() - hasher.Write([]byte(text)) + for _, attr := range attrs { + hasher.Write([]byte(attr)) + } return fmt.Sprintf("%x", hasher.Sum(nil)) } -func FSCgrId(uuid string) string { - return SHA1(uuid) -} - func NewTPid() string { - return SHA1(GenUUID()) + return Sha1(GenUUID()) } // helper function for uuid generation diff --git a/utils/rawcdr.go b/utils/rawcdr.go index 4577e5e62..efe077c69 100644 --- a/utils/rawcdr.go +++ b/utils/rawcdr.go @@ -22,7 +22,7 @@ import ( "time" ) -var PrimaryCdrFields []string = []string{ACCID, CDRHOST, CDRSOURCE, REQTYPE, DIRECTION, TENANT, TOR, ACCOUNT, SUBJECT, DESTINATION, ANSWER_TIME, DURATION} +var PrimaryCdrFields []string = []string{ACCID, CDRHOST, CDRSOURCE, REQTYPE, DIRECTION, TENANT, TOR, ACCOUNT, SUBJECT, DESTINATION, SETUP_TIME, ANSWER_TIME, DURATION} // RawCDR is the type containing all the original CDR fields, needs it as it is for later usage type RawCDR interface { diff --git a/utils/storedcdr.go b/utils/storedcdr.go index 7e69e2a9e..b6670dd0a 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -55,7 +55,7 @@ func NewStoredCdrFromRawCDR(rawcdr RawCDR) (*StoredCdr, error) { // Rated CDR as extracted from StorDb. Kinda standard of internal CDR, complies to CDR interface also type StoredCdr struct { CgrId string - OrderId int64 // Stor order id used as export order id + OrderId int64 // Stor order id used as export order id AccId string CdrHost string CdrSource string diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index 0226e76fc..1368efe01 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -31,11 +31,12 @@ func TestStoredCdrInterfaces(t *testing.T) { func TestNewStoredCdrFromRawCDR(t *testing.T) { cgrCdr := CgrCdr{"accid": "dsafdsaf", "cdrhost": "192.168.1.1", "cdrsource": "internal_test", "reqtype": "rated", "direction": "*out", "tenant": "cgrates.org", "tor": "call", - "account": "1001", "subject": "1001", "destination": "1002", "answer_time": "2013-11-07T08:42:26Z", "duration": "10", + "account": "1001", "subject": "1001", "destination": "1002", "setup_time": "2013-11-07T08:42:20Z", "answer_time": "2013-11-07T08:42:26Z", "duration": "10", "field_extr1": "val_extr1", "fieldextr2": "valextr2"} - expctRtCdr := &StoredCdr{CgrId: FSCgrId(cgrCdr["accid"]), AccId: cgrCdr["accid"], CdrHost: cgrCdr["cdrhost"], CdrSource: cgrCdr["cdrsource"], ReqType: cgrCdr["reqtype"], + setupTime, _ := ParseTimeDetectLayout(cgrCdr["setup_time"]) + expctRtCdr := &StoredCdr{CgrId: Sha1(cgrCdr["accid"], setupTime.String()), AccId: cgrCdr["accid"], CdrHost: cgrCdr["cdrhost"], CdrSource: cgrCdr["cdrsource"], ReqType: cgrCdr["reqtype"], Direction: cgrCdr["direction"], Tenant: cgrCdr["tenant"], TOR: cgrCdr["tor"], Account: cgrCdr["account"], Subject: cgrCdr["subject"], - Destination: cgrCdr["destination"], AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(10) * time.Second, + Destination: cgrCdr["destination"], SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, MediationRunId: DEFAULT_RUNID, Cost: -1} if rt, err := NewStoredCdrFromRawCDR(cgrCdr); err != nil { t.Error(err) @@ -45,11 +46,11 @@ func TestNewStoredCdrFromRawCDR(t *testing.T) { } func TestStoredCdrFields(t *testing.T) { - ratedCdr := StoredCdr{CgrId: FSCgrId("dsafdsaf"), AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", - TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", AnswerTime: time.Unix(1383813746, 0), Duration: 10, + ratedCdr := StoredCdr{CgrId: Sha1("dsafdsaf", time.Unix(1383813746, 0).String()), AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", + TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813746, 0), AnswerTime: time.Unix(1383813746, 0), Duration: 10, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } - if ratedCdr.GetCgrId() != "b18944ef4dc618569f24c27b9872827a242bad0c" { + if ratedCdr.GetCgrId() != Sha1("dsafdsaf", time.Unix(1383813746, 0).String()) { t.Error("Error parsing cdr: ", ratedCdr) } if ratedCdr.GetAccId() != "dsafdsaf" { @@ -79,6 +80,11 @@ func TestStoredCdrFields(t *testing.T) { if ratedCdr.GetReqType() != RATED { t.Error("Error parsing cdr: ", ratedCdr) } + setupTime, _ := ratedCdr.GetSetupTime() + expectedSTime, _ := time.Parse(time.RFC3339, "2013-11-07T08:42:26Z") + if setupTime.UTC() != expectedSTime { + t.Error("Error parsing cdr: ", ratedCdr) + } answerTime, _ := ratedCdr.GetAnswerTime() expectedATime, _ := time.Parse(time.RFC3339, "2013-11-07T08:42:26Z") if answerTime.UTC() != expectedATime { @@ -100,8 +106,8 @@ func TestStoredCdrFields(t *testing.T) { } func TestAsRawCdrHttpForm(t *testing.T) { - ratedCdr := StoredCdr{CgrId: FSCgrId("dsafdsaf"), AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", - TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), + ratedCdr := StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()), AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", + TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), Duration: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } cdrForm := ratedCdr.AsRawCdrHttpForm() @@ -135,6 +141,9 @@ func TestAsRawCdrHttpForm(t *testing.T) { if cdrForm.Get(DESTINATION) != ratedCdr.Destination { t.Errorf("Expected: %s, received: %s", ratedCdr.Destination, cdrForm.Get(DESTINATION)) } + if cdrForm.Get(SETUP_TIME) != "2013-11-07 08:42:20 +0000 UTC" { + t.Errorf("Expected: %s, received: %s", "2013-11-07 08:42:26 +0000 UTC", cdrForm.Get(SETUP_TIME)) + } if cdrForm.Get(ANSWER_TIME) != "2013-11-07 08:42:26 +0000 UTC" { t.Errorf("Expected: %s, received: %s", "2013-11-07 08:42:26 +0000 UTC", cdrForm.Get(ANSWER_TIME)) } @@ -150,8 +159,8 @@ func TestAsRawCdrHttpForm(t *testing.T) { } func TestExportFieldValue(t *testing.T) { - cdr := StoredCdr{CgrId: FSCgrId("dsafdsaf"), OrderId: 123, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", - TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID, + cdr := StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", + TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID, Duration: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } if cdr.ExportFieldValue(CGRID) != cdr.CgrId || @@ -166,7 +175,7 @@ func TestExportFieldValue(t *testing.T) { cdr.ExportFieldValue(ACCOUNT) != cdr.Account || cdr.ExportFieldValue(SUBJECT) != cdr.Subject || cdr.ExportFieldValue(DESTINATION) != cdr.Destination || - cdr.ExportFieldValue(SETUP_TIME) != "0001-01-01 00:00:00 +0000 UTC" || + cdr.ExportFieldValue(SETUP_TIME) != cdr.SetupTime.String() || cdr.ExportFieldValue(ANSWER_TIME) != cdr.AnswerTime.String() || cdr.ExportFieldValue(DURATION) != "10" || cdr.ExportFieldValue(MEDI_RUNID) != cdr.MediationRunId ||