From ff7f4cac092748bdf9ef7c994fe087180debf8c7 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 1 Jul 2014 18:27:12 +0200 Subject: [PATCH 01/25] Fix CDRExporter to avoid writing empty fwv files, cgr-loader now orders rating recache in case of flushdb option used --- cdre/cdrexporter.go | 4 ++++ cmd/cgr-loader/cgr-loader.go | 3 +++ 2 files changed, 7 insertions(+) diff --git a/cdre/cdrexporter.go b/cdre/cdrexporter.go index c62fa5e5c..cd7b3760a 100644 --- a/cdre/cdrexporter.go +++ b/cdre/cdrexporter.go @@ -442,6 +442,10 @@ func (cdre *CdrExporter) writeCsv(csvWriter *csv.Writer) error { // General method to write the content out to a file func (cdre *CdrExporter) WriteToFile(filePath string) error { + if cdre.numberOfRecords == 0 { // Not writing the file in case of no CDRs to be exported + engine.Logger.Err(" Not writing file out since there are no records to export") + return nil + } fileOut, err := os.Create(filePath) if err != nil { return err diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index dfaa5f87e..749bdf20b 100644 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -198,6 +198,9 @@ func main() { if *verbose { log.Print("Reloading cache") } + if *flush { + dstIds, rplIds, rpfIds, rpAliases, lcrIds = nil, nil, nil, nil, nil // Should reload all these on flush + } if err = rater.Call("ApierV1.ReloadCache", utils.ApiReloadCache{dstIds, rplIds, rpfIds, actIds, shgIds, rpAliases, accAliases, lcrIds, dcs}, &reply); err != nil { log.Fatalf("Got error on cache reload: %s", err.Error()) } From a4b3d94431b2c7bb736afe4928e776c16cec4213 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 2 Jul 2014 19:39:19 +0200 Subject: [PATCH 02/25] Derived charging - fix nil pointer dereference in case of empty fields on fork cdr, cdrexporter correctly returning now empty file path in case of no cdrs to be exported --- apier/cdre.go | 4 ++++ cdre/cdrexporter.go | 8 ++++---- utils/storedcdr.go | 39 ++++++++++++++++++++++++++++++++++++++- utils/storedcdr_test.go | 7 +++++++ 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/apier/cdre.go b/apier/cdre.go index 720c9417d..7018621eb 100644 --- a/apier/cdre.go +++ b/apier/cdre.go @@ -129,6 +129,10 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E if err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } + if cdrexp.TotalExportedCdrs() == 0 { + *reply = utils.ExportedFileCdrs{ExportedFilePath: ""} + return nil + } if err := cdrexp.WriteToFile(filePath); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } diff --git a/cdre/cdrexporter.go b/cdre/cdrexporter.go index cd7b3760a..faf4e10b7 100644 --- a/cdre/cdrexporter.go +++ b/cdre/cdrexporter.go @@ -442,10 +442,6 @@ func (cdre *CdrExporter) writeCsv(csvWriter *csv.Writer) error { // General method to write the content out to a file func (cdre *CdrExporter) WriteToFile(filePath string) error { - if cdre.numberOfRecords == 0 { // Not writing the file in case of no CDRs to be exported - engine.Logger.Err(" Not writing file out since there are no records to export") - return nil - } fileOut, err := os.Create(filePath) if err != nil { return err @@ -482,6 +478,10 @@ func (cdre *CdrExporter) TotalCost() float64 { return cdre.totalCost } +func (cdre *CdrExporter) TotalExportedCdrs() int { + return cdre.numberOfRecords +} + // Return successfully exported CgrIds func (cdre *CdrExporter) PositiveExports() []string { return cdre.positiveExports diff --git a/utils/storedcdr.go b/utils/storedcdr.go index aedd8c057..ca821bda7 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -163,34 +163,71 @@ func (storedCdr *StoredCdr) AsHttpForm() url.Values { // Used in mediation, primaryMandatory marks whether missing field out of request represents error or can be ignored func (storedCdr *StoredCdr) ForkCdr(runId string, reqTypeFld, directionFld, tenantFld, categFld, accountFld, subjectFld, destFld, setupTimeFld, answerTimeFld, durationFld *RSRField, extraFlds []*RSRField, primaryMandatory bool) (*StoredCdr, error) { - // MetaDefault will automatically be converted to their standard values + // A more elegant solution for the future to fix + /*for _, fld := range []*RSRField{reqTypeFld, directionFld, tenantFld, categFld, accountFld, subjectFld, destFld, setupTimeFld, answerTimeFld, durationFld} { + if fld == nil { + tmp, _ := NewRSRField(META_DEFAULT) + *fld = *tmp + } + } + */ + if reqTypeFld == nil { + reqTypeFld, _ = NewRSRField(META_DEFAULT) + } if reqTypeFld.Id == META_DEFAULT { reqTypeFld.Id = REQTYPE } + if directionFld == nil { + directionFld, _ = NewRSRField(META_DEFAULT) + } if directionFld.Id == META_DEFAULT { directionFld.Id = DIRECTION } + if tenantFld == nil { + tenantFld, _ = NewRSRField(META_DEFAULT) + } if tenantFld.Id == META_DEFAULT { tenantFld.Id = TENANT } + if categFld == nil { + categFld, _ = NewRSRField(META_DEFAULT) + } if categFld.Id == META_DEFAULT { categFld.Id = CATEGORY } + if accountFld == nil { + accountFld, _ = NewRSRField(META_DEFAULT) + } if accountFld.Id == META_DEFAULT { accountFld.Id = ACCOUNT } + if subjectFld == nil { + subjectFld, _ = NewRSRField(META_DEFAULT) + } if subjectFld.Id == META_DEFAULT { subjectFld.Id = SUBJECT } + if destFld == nil { + destFld, _ = NewRSRField(META_DEFAULT) + } if destFld.Id == META_DEFAULT { destFld.Id = DESTINATION } + if setupTimeFld == nil { + setupTimeFld, _ = NewRSRField(META_DEFAULT) + } if setupTimeFld.Id == META_DEFAULT { setupTimeFld.Id = SETUP_TIME } + if answerTimeFld == nil { + answerTimeFld, _ = NewRSRField(META_DEFAULT) + } if answerTimeFld.Id == META_DEFAULT { answerTimeFld.Id = ANSWER_TIME } + if durationFld == nil { + durationFld, _ = NewRSRField(META_DEFAULT) + } if durationFld.Id == META_DEFAULT { durationFld.Id = USAGE } diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index 4f1590703..baf8018a3 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -271,6 +271,13 @@ func TestStoredCdrForkCdrFromMetaDefaults(t *testing.T) { if !reflect.DeepEqual(expctCdr, cdrOut) { t.Errorf("Expected: %v, received: %v", expctCdr, cdrOut) } + // Should also accept nil as defaults + if cdrOut, err := storCdr.ForkCdr("wholesale_run", nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + []*RSRField{&RSRField{Id: "field_extr1"}, &RSRField{Id: "fieldextr2"}}, true); err != nil { + t.Fatal("Unexpected error received", err) + } else if !reflect.DeepEqual(expctCdr, cdrOut) { + t.Errorf("Expected: %v, received: %v", expctCdr, cdrOut) + } } func TestStoredCdrAsCgrCdrOut(t *testing.T) { From a014ef840181aa0092f03f464901fa45635ef6a0 Mon Sep 17 00:00:00 2001 From: DanB Date: Thu, 3 Jul 2014 19:50:14 +0200 Subject: [PATCH 03/25] Cdrc to accept static fields in case of .csv files --- cdrc/cdrc.go | 17 +++--- cdrc/cdrc_test.go | 125 +++++++++++++++++++++++++++++++++++++++++++- utils/coreutils.go | 4 +- utils/storedcdr.go | 8 --- utils/utils_test.go | 4 +- 5 files changed, 138 insertions(+), 20 deletions(-) diff --git a/cdrc/cdrc.go b/cdrc/cdrc.go index 6facf7271..f485dc61f 100644 --- a/cdrc/cdrc.go +++ b/cdrc/cdrc.go @@ -28,6 +28,7 @@ import ( "os" "path" "strconv" + "strings" "time" "unicode/utf8" @@ -85,16 +86,20 @@ func (self *Cdrc) Run() error { } // Takes the record out of csv and turns it into http form which can be posted -func (self *Cdrc) recordForkCdr(record []string) (*utils.StoredCdr, error) { +func (self *Cdrc) recordToStoredCdr(record []string) (*utils.StoredCdr, error) { storedCdr := &utils.StoredCdr{CdrHost: "0.0.0.0", CdrSource: self.cdrSourceId, ExtraFields: make(map[string]string), Cost: -1} var err error for cfgFieldName, cfgFieldRSR := range self.cdrFields { var fieldVal string if utils.IsSliceMember([]string{CSV, FS_CSV}, self.cdrType) { - if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx { - return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cfgFieldName) - } else { - fieldVal = cfgFieldRSR.ParseValue(record[cfgFieldIdx]) + if strings.HasPrefix(cfgFieldRSR.Id, utils.STATIC_VALUE_PREFIX) { + fieldVal = cfgFieldRSR.ParseValue("PLACEHOLDER") + } else { // Dynamic value extracted using index + if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx { + return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cfgFieldName) + } else { + fieldVal = cfgFieldRSR.ParseValue(record[cfgFieldIdx]) + } } } else { // Modify here when we add more supported cdr formats fieldVal = "UNKNOWN" @@ -199,7 +204,7 @@ func (self *Cdrc) processFile(filePath string) error { engine.Logger.Err(fmt.Sprintf(" Error in csv file: %s", err.Error())) continue // Other csv related errors, ignore } - storedCdr, err := self.recordForkCdr(record) + storedCdr, err := self.recordToStoredCdr(record) if err != nil { engine.Logger.Err(fmt.Sprintf(" Error in csv file: %s", err.Error())) continue diff --git a/cdrc/cdrc_test.go b/cdrc/cdrc_test.go index c1c59a77b..7874c1e59 100644 --- a/cdrc/cdrc_test.go +++ b/cdrc/cdrc_test.go @@ -19,9 +19,13 @@ along with this program. If not, see package cdrc import ( + //"bytes" + //"encoding/csv" + //"fmt" "github.com/cgrates/cgrates/cdrs" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" + //"io" "reflect" "testing" "time" @@ -35,13 +39,13 @@ func TestRecordForkCdr(t *testing.T) { cdrc := &Cdrc{cgrConfig.CdrcCdrs, cgrConfig.CdrcCdrType, cgrConfig.CdrcCdrInDir, cgrConfig.CdrcCdrOutDir, cgrConfig.CdrcSourceId, cgrConfig.CdrcRunDelay, csvSepRune, cgrConfig.CdrcCdrFields, new(cdrs.CDRS), nil} cdrRow := []string{"firstField", "secondField"} - _, err := cdrc.recordForkCdr(cdrRow) + _, err := cdrc.recordToStoredCdr(cdrRow) if err == nil { t.Error("Failed to corectly detect missing fields from record") } cdrRow = []string{"ignored", "ignored", utils.VOICE, "acc1", "prepaid", "*out", "cgrates.org", "call", "1001", "1001", "+4986517174963", "2013-02-03 19:50:00", "2013-02-03 19:54:00", "62000000000", "supplier1", "172.16.1.1"} - rtCdr, err := cdrc.recordForkCdr(cdrRow) + rtCdr, err := cdrc.recordToStoredCdr(cdrRow) if err != nil { t.Error("Failed to parse CDR in rated cdr", err) } @@ -68,3 +72,120 @@ func TestRecordForkCdr(t *testing.T) { t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr) } } + +/* +func TestDnTdmCdrs(t *testing.T) { + tdmCdrs := ` +49773280254,0049LN130676000285,N_IP_0676_00-Internet 0676 WRAP 13,02.07.2014 15:24:40,02.07.2014 15:24:40,1,25,Peak,0.000000,49DE13 +49893252121,0049651515477,N_MO_MRAP_00-WRAP Mobile,02.07.2014 15:24:41,02.07.2014 15:24:41,1,8,Peak,0.003920,49651 +49497361022,0049LM0409005226,N_MO_MTMB_00-RW-Mobile,02.07.2014 15:24:41,02.07.2014 15:24:41,1,43,Peak,0.021050,49MTMB +` + cgrConfig, _ := config.NewDefaultCGRConfig() + eCdrs := []*utils.StoredCdr{ + &utils.StoredCdr{ + CgrId: utils.Sha1("49773280254", time.Date(2014, 7, 2, 15, 24, 40, 0, time.UTC).String()), + TOR: utils.VOICE, + AccId: "49773280254", + CdrHost: "0.0.0.0", + CdrSource: cgrConfig.CdrcSourceId, + ReqType: "rated", + Direction: "*out", + Tenant: "sip.test.deanconnect.nl", + Category: "call", + Account: "+49773280254", + Subject: "+49773280254", + Destination: "+49676000285", + SetupTime: time.Date(2014, 7, 2, 15, 24, 40, 0, time.UTC), + AnswerTime: time.Date(2014, 7, 2, 15, 24, 40, 0, time.UTC), + Usage: time.Duration(25) * time.Second, + Cost: -1, + }, + &utils.StoredCdr{ + CgrId: utils.Sha1("49893252121", time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC).String()), + TOR: utils.VOICE, + AccId: "49893252121", + CdrHost: "0.0.0.0", + CdrSource: cgrConfig.CdrcSourceId, + ReqType: "rated", + Direction: "*out", + Tenant: "sip.test.deanconnect.nl", + Category: "call", + Account: "+49893252121", + Subject: "+49893252121", + Destination: "+49651515477", + SetupTime: time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC), + AnswerTime: time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC), + Usage: time.Duration(8) * time.Second, + Cost: -1, + }, + &utils.StoredCdr{ + CgrId: utils.Sha1("49497361022", time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC).String()), + TOR: utils.VOICE, + AccId: "49497361022", + CdrHost: "0.0.0.0", + CdrSource: cgrConfig.CdrcSourceId, + ReqType: "rated", + Direction: "*out", + Tenant: "sip.test.deanconnect.nl", + Category: "call", + Account: "+49497361022", + Subject: "+49497361022", + Destination: "+499005226", + SetupTime: time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC), + AnswerTime: time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC), + Usage: time.Duration(43) * time.Second, + Cost: -1, + }, + } + torFld, _ := utils.NewRSRField("^*voice") + acntFld, _ := utils.NewRSRField(`~0:s/^([1-9]\d+)$/+$1/`) + reqTypeFld, _ := utils.NewRSRField("^rated") + dirFld, _ := utils.NewRSRField("^*out") + tenantFld, _ := utils.NewRSRField("^sip.test.deanconnect.nl") + categFld, _ := utils.NewRSRField("^call") + dstFld, _ := utils.NewRSRField(`~1:s/^00(\d+)(?:[a-zA-Z].{3})*0*([1-9]\d+)$/+$1$2/`) + usageFld, _ := utils.NewRSRField(`~6:s/^(\d+)$/${1}s/`) + cgrConfig.CdrcCdrFields = map[string]*utils.RSRField{ + utils.TOR: torFld, + utils.ACCID: &utils.RSRField{Id: "0"}, + utils.REQTYPE: reqTypeFld, + utils.DIRECTION: dirFld, + utils.TENANT: tenantFld, + utils.CATEGORY: categFld, + utils.ACCOUNT: acntFld, + utils.SUBJECT: acntFld, + utils.DESTINATION: dstFld, + utils.SETUP_TIME: &utils.RSRField{Id: "4"}, + utils.ANSWER_TIME: &utils.RSRField{Id: "4"}, + utils.USAGE: usageFld, + } + cdrc := &Cdrc{cgrConfig.CdrcCdrs, cgrConfig.CdrcCdrType, cgrConfig.CdrcCdrInDir, cgrConfig.CdrcCdrOutDir, cgrConfig.CdrcSourceId, cgrConfig.CdrcRunDelay, ',', + cgrConfig.CdrcCdrFields, new(cdrs.CDRS), nil} + cdrsContent := bytes.NewReader([]byte(tdmCdrs)) + csvReader := csv.NewReader(cdrsContent) + cdrs := make([]*utils.StoredCdr, 0) + for { + cdrCsv, err := csvReader.Read() + if err != nil && err == io.EOF { + break // End of file + } else if err != nil { + t.Error("Unexpected error:", err) + } + if cdr, err := cdrc.recordToStoredCdr(cdrCsv); err != nil { + t.Error("Unexpected error: ", err) + } else { + cdrs = append(cdrs, cdr) + } + } + if !reflect.DeepEqual(eCdrs, cdrs) { + for _, ecdr := range eCdrs { + fmt.Printf("Cdr expected: %+v\n", ecdr) + } + for _, cdr := range cdrs { + fmt.Printf("Cdr processed: %+v\n", cdr) + } + t.Errorf("Expecting: %+v, received: %+v", eCdrs, cdrs) + } + +} +*/ diff --git a/utils/coreutils.go b/utils/coreutils.go index dcffef59e..46a6632ae 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -118,7 +118,7 @@ func ParseTimeDetectLayout(tmStr string) (time.Time, error) { fsTimestamp := regexp.MustCompile(`^\d{16}$`) unixTimestampRule := regexp.MustCompile(`^\d{10}$`) oneLineTimestampRule := regexp.MustCompile(`^\d{14}$`) - twoSpaceTimestampRule := regexp.MustCompile(`^\d{2}\.\d{2}.\d{4}\s{2}\d{2}:\d{2}:\d{2}$`) + oneSpaceTimestampRule := regexp.MustCompile(`^\d{2}\.\d{2}.\d{4}\s{1}\d{2}:\d{2}:\d{2}$`) switch { case rfc3339Rule.MatchString(tmStr): return time.Parse(time.RFC3339, tmStr) @@ -142,7 +142,7 @@ func ParseTimeDetectLayout(tmStr string) (time.Time, error) { return nilTime, nil case oneLineTimestampRule.MatchString(tmStr): return time.Parse("20060102150405", tmStr) - case twoSpaceTimestampRule.MatchString(tmStr): + case oneSpaceTimestampRule.MatchString(tmStr): return time.Parse("02.01.2006 15:04:05", tmStr) } return nilTime, errors.New("Unsupported time format") diff --git a/utils/storedcdr.go b/utils/storedcdr.go index ca821bda7..2c2d03a5a 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -163,14 +163,6 @@ func (storedCdr *StoredCdr) AsHttpForm() url.Values { // Used in mediation, primaryMandatory marks whether missing field out of request represents error or can be ignored func (storedCdr *StoredCdr) ForkCdr(runId string, reqTypeFld, directionFld, tenantFld, categFld, accountFld, subjectFld, destFld, setupTimeFld, answerTimeFld, durationFld *RSRField, extraFlds []*RSRField, primaryMandatory bool) (*StoredCdr, error) { - // A more elegant solution for the future to fix - /*for _, fld := range []*RSRField{reqTypeFld, directionFld, tenantFld, categFld, accountFld, subjectFld, destFld, setupTimeFld, answerTimeFld, durationFld} { - if fld == nil { - tmp, _ := NewRSRField(META_DEFAULT) - *fld = *tmp - } - } - */ if reqTypeFld == nil { reqTypeFld, _ = NewRSRField(META_DEFAULT) } diff --git a/utils/utils_test.go b/utils/utils_test.go index e71f1a6e5..e904b9633 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -211,8 +211,8 @@ func TestParseTimeDetectLayout(t *testing.T) { } else if !olTm.Equal(expectedTime) { t.Errorf("Unexpected time parsed: %v, expecting: %v", olTm, expectedTime) } - twoSpaceTmStr := "08.04.2014 22:14:29" - tsTm, err := ParseTimeDetectLayout(twoSpaceTmStr) + oneSpaceTmStr := "08.04.2014 22:14:29" + tsTm, err := ParseTimeDetectLayout(oneSpaceTmStr) expectedTime = time.Date(2014, 4, 8, 22, 14, 29, 0, time.UTC) if err != nil { t.Error(err) From 9b37e97cc29866625f5a1a9864357f2145aa8df1 Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 4 Jul 2014 20:05:27 +0200 Subject: [PATCH 04/25] RunFilter in DerivedChargers, ParseEventValue accepting RSRField in FreeSWITCH event --- config/config_test.go | 14 +- config/helpers.go | 30 +-- config/helpers_test.go | 10 + config/test_data.txt | 1 + data/conf/cgrates.cfg | 1 + mediator/mediator.go | 5 + sessionmanager/event.go | 2 + sessionmanager/fsevent.go | 46 ++++ sessionmanager/fsevent_test.go | 325 +++++++++++++++++------------ sessionmanager/fssessionmanager.go | 4 + utils/derivedchargers.go | 36 ++-- utils/derivedchargers_test.go | 10 +- 12 files changed, 314 insertions(+), 170 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 03c1aece4..dc90fd857 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -250,7 +250,7 @@ func TestConfigFromFile(t *testing.T) { eCfg.FreeswitchServer = "test" eCfg.FreeswitchPass = "test" eCfg.FreeswitchReconnects = 99 - eCfg.DerivedChargers = utils.DerivedChargers{&utils.DerivedCharger{RunId: "test", ReqTypeField: "test", DirectionField: "test", TenantField: "test", + eCfg.DerivedChargers = utils.DerivedChargers{&utils.DerivedCharger{RunId: "test", RunFilter: "", ReqTypeField: "test", DirectionField: "test", TenantField: "test", CategoryField: "test", AccountField: "test", SubjectField: "test", DestinationField: "test", SetupTimeField: "test", AnswerTimeField: "test", UsageField: "test"}} eCfg.CombinedDerivedChargers = true eCfg.HistoryAgentEnabled = true @@ -264,7 +264,13 @@ func TestConfigFromFile(t *testing.T) { eCfg.MailerFromAddr = "test" if !reflect.DeepEqual(cfg, eCfg) { t.Log(eCfg) + for _, eDC := range eCfg.DerivedChargers { + fmt.Printf("ExpectDerivedChargers: %+v\n", eDC) + } t.Log(cfg) + for _, eDC := range cfg.DerivedChargers { + fmt.Printf("DerivedChargers: %+v\n", eDC) + } t.Error("Loading of configuration from file failed!") } } @@ -288,12 +294,6 @@ extra_fields = ~effective_caller_id_number:s/(\d+)/+$1/ t.Errorf("Unexpected value for config CdrsExtraFields: %v", cfg.CDRSExtraFields) } eFieldsCfg = []byte(`[cdrs] -extra_fields = extr1,extr2, -`) - if _, err := NewCGRConfigFromBytes(eFieldsCfg); err == nil { - t.Error("Failed to detect empty field in the end of extra fields defition") - } - eFieldsCfg = []byte(`[cdrs] extra_fields = extr1,~extr2:s/x.+/ `) if _, err := NewCGRConfigFromBytes(eFieldsCfg); err == nil { diff --git a/config/helpers.go b/config/helpers.go index 580ebb7f6..02fad02ce 100644 --- a/config/helpers.go +++ b/config/helpers.go @@ -29,15 +29,11 @@ import ( // Adds support for slice values in config func ConfigSlice(cfgVal string) ([]string, error) { - cfgValStrs := strings.Split(cfgVal, ",") // If need arrises, we can make the separator configurable - if len(cfgValStrs) == 1 && cfgValStrs[0] == "" { // Prevents returning iterable with empty value - return []string{}, nil - } + cfgValStrs := strings.Split(cfgVal, ",") // If need arrises, we can make the separator configurable for idx, elm := range cfgValStrs { - if elm == "" { //One empty element is presented when splitting empty string - return nil, errors.New("Empty values in config slice") - - } + //if elm == "" { //One empty element is presented when splitting empty string + // return nil, errors.New("Empty values in config slice") + //} cfgValStrs[idx] = strings.TrimSpace(elm) // By default spaces are not removed so we do it here to avoid unpredicted results in config } return cfgValStrs, nil @@ -50,10 +46,6 @@ func ParseRSRFields(configVal string) ([]*utils.RSRField, error) { } rsrFields := make([]*utils.RSRField, len(cfgValStrs)) for idx, cfgValStr := range cfgValStrs { - if len(cfgValStr) == 0 { //One empty element is presented when splitting empty string - return nil, errors.New("Empty values in config slice") - - } if rsrField, err := utils.NewRSRField(cfgValStr); err != nil { return nil, err } else { @@ -65,11 +57,15 @@ func ParseRSRFields(configVal string) ([]*utils.RSRField, error) { // Parse the configuration file and returns utils.DerivedChargers instance if no errors func ParseCfgDerivedCharging(c *conf.ConfigFile) (dcs utils.DerivedChargers, err error) { - var runIds, reqTypeFlds, directionFlds, tenantFlds, torFlds, acntFlds, subjFlds, dstFlds, sTimeFlds, aTimeFlds, durFlds []string + var runIds, runFilters, reqTypeFlds, directionFlds, tenantFlds, torFlds, acntFlds, subjFlds, dstFlds, sTimeFlds, aTimeFlds, durFlds []string cfgVal, _ := c.GetString("derived_charging", "run_ids") if runIds, err = ConfigSlice(cfgVal); err != nil { return nil, err } + cfgVal, _ = c.GetString("derived_charging", "run_filters") + if runFilters, err = ConfigSlice(cfgVal); err != nil { + return nil, err + } cfgVal, _ = c.GetString("derived_charging", "reqtype_fields") if reqTypeFlds, err = ConfigSlice(cfgVal); err != nil { return nil, err @@ -111,7 +107,8 @@ func ParseCfgDerivedCharging(c *conf.ConfigFile) (dcs utils.DerivedChargers, err return nil, err } // We need all to be the same length - if len(reqTypeFlds) != len(runIds) || + if len(runFilters) != len(runIds) || + len(reqTypeFlds) != len(runIds) || len(directionFlds) != len(runIds) || len(tenantFlds) != len(runIds) || len(torFlds) != len(runIds) || @@ -125,8 +122,11 @@ func ParseCfgDerivedCharging(c *conf.ConfigFile) (dcs utils.DerivedChargers, err } // Create the individual chargers and append them to the final instance dcs = make(utils.DerivedChargers, 0) + if len(runIds) == 1 && len(runIds[0]) == 0 { // Avoid iterating on empty runid + return dcs, nil + } for runIdx, runId := range runIds { - dc, err := utils.NewDerivedCharger(runId, reqTypeFlds[runIdx], directionFlds[runIdx], tenantFlds[runIdx], torFlds[runIdx], + dc, err := utils.NewDerivedCharger(runId, runFilters[runIdx], reqTypeFlds[runIdx], directionFlds[runIdx], tenantFlds[runIdx], torFlds[runIdx], acntFlds[runIdx], subjFlds[runIdx], dstFlds[runIdx], sTimeFlds[runIdx], aTimeFlds[runIdx], durFlds[runIdx]) if err != nil { return nil, err diff --git a/config/helpers_test.go b/config/helpers_test.go index 741a7065c..13aaeff36 100644 --- a/config/helpers_test.go +++ b/config/helpers_test.go @@ -26,6 +26,15 @@ import ( "github.com/cgrates/cgrates/utils" ) +func TestConfigSlice(t *testing.T) { + eCS := []string{"", ""} + if cs, err := ConfigSlice(" , "); err != nil { + t.Error("Unexpected error: ", err) + } else if !reflect.DeepEqual(eCS, cs) { + t.Errorf("Expecting: %v, received: %v", eCS, cs) + } +} + func TestParseRSRFields(t *testing.T) { fields := `host,~sip_redirected_to:s/sip:\+49(\d+)@/0$1/,destination` expectParsedFields := []*utils.RSRField{&utils.RSRField{Id: "host"}, @@ -41,6 +50,7 @@ func TestParseRSRFields(t *testing.T) { func TestParseCfgDerivedCharging(t *testing.T) { eFieldsCfg := []byte(`[derived_charging] run_ids = run1, run2 +run_filters =, reqtype_fields = test1, test2 direction_fields = test1, test2 tenant_fields = test1, test2 diff --git a/config/test_data.txt b/config/test_data.txt index bd9e050d9..77104a741 100644 --- a/config/test_data.txt +++ b/config/test_data.txt @@ -100,6 +100,7 @@ reconnects = 99 # Number of attempts on connect failure. [derived_charging] run_ids = test # Identifiers of additional sessions control. +run_filters = # No filters applied reqtype_fields = test # Name of request type fields to be used during additional sessions control <""|*default|field_name>. direction_fields = test # Name of direction fields to be used during additional sessions control <""|*default|field_name>. tenant_fields = test # Name of tenant fields to be used during additional sessions control <""|*default|field_name>. diff --git a/data/conf/cgrates.cfg b/data/conf/cgrates.cfg index 70da87d56..2f336a50d 100644 --- a/data/conf/cgrates.cfg +++ b/data/conf/cgrates.cfg @@ -104,6 +104,7 @@ [derived_charging] # run_ids = # Identifiers of additional sessions control. +# run_filters = # List of cdr field filters for each run. # reqtype_fields = # Name of request type fields to be used during additional sessions control <""|*default|field_name>. # direction_fields = # Name of direction fields to be used during additional sessions control <""|*default|field_name>. # tenant_fields = # Name of tenant fields to be used during additional sessions control <""|*default|field_name>. diff --git a/mediator/mediator.go b/mediator/mediator.go index 09997e17b..6dc813935 100644 --- a/mediator/mediator.go +++ b/mediator/mediator.go @@ -120,6 +120,11 @@ func (self *Mediator) RateCdr(storedCdr *utils.StoredCdr) error { return errors.New(errText) } for _, dc := range dcs { + dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) + if dcRunFilter != nil && storedCdr.FieldAsString(&utils.RSRField{Id: dcRunFilter.Id}) != storedCdr.FieldAsString(dcRunFilter) { + engine.Logger.Info(fmt.Sprintf("Ignoring DerivedCharger with id %s due to non matching filter", dc.RunId)) + continue + } dcReqTypeFld, _ := utils.NewRSRField(dc.ReqTypeField) dcDirFld, _ := utils.NewRSRField(dc.DirectionField) dcTenantFld, _ := utils.NewRSRField(dc.TenantField) diff --git a/sessionmanager/event.go b/sessionmanager/event.go index c39c3a8b8..4b4f1c4e3 100644 --- a/sessionmanager/event.go +++ b/sessionmanager/event.go @@ -19,6 +19,7 @@ along with this program. If not, see package sessionmanager import ( + "github.com/cgrates/cgrates/utils" "time" ) @@ -40,4 +41,5 @@ type Event interface { GetEndTime() (time.Time, error) GetDuration(string) (time.Duration, error) MissingParameter() bool + ParseEventValue(*utils.RSRField) string } diff --git a/sessionmanager/fsevent.go b/sessionmanager/fsevent.go index 0d54d42b6..3f75d983b 100644 --- a/sessionmanager/fsevent.go +++ b/sessionmanager/fsevent.go @@ -20,6 +20,7 @@ package sessionmanager import ( "fmt" + "strconv" "strings" "time" @@ -203,3 +204,48 @@ func (fsev FSEvent) GetDuration(fieldName string) (dur time.Duration, err error) } return utils.ParseDurationWithSecs(durStr) } + +// Used in derived charging and sittuations when we need to run regexp on fields +func (fsev FSEvent) ParseEventValue(rsrFld *utils.RSRField) string { + switch rsrFld.Id { + case utils.CGRID: + return rsrFld.ParseValue(fsev.GetCgrId()) + case utils.TOR: + return rsrFld.ParseValue(utils.VOICE) + case utils.ACCID: + return rsrFld.ParseValue(fsev.GetUUID()) + case utils.CDRHOST: + return rsrFld.ParseValue(fsev["FreeSWITCH-IPv4"]) + case utils.CDRSOURCE: + return rsrFld.ParseValue("FS_EVENT") + case utils.REQTYPE: + return rsrFld.ParseValue(fsev.GetReqType("")) + case utils.DIRECTION: + return rsrFld.ParseValue(fsev.GetDirection("")) + case utils.TENANT: + return rsrFld.ParseValue(fsev.GetTenant("")) + case utils.CATEGORY: + return rsrFld.ParseValue(fsev.GetCategory("")) + case utils.ACCOUNT: + return rsrFld.ParseValue(fsev.GetAccount("")) + case utils.SUBJECT: + return rsrFld.ParseValue(fsev.GetSubject("")) + case utils.DESTINATION: + return rsrFld.ParseValue(fsev.GetDestination("")) + case utils.SETUP_TIME: + st, _ := fsev.GetSetupTime("") + return rsrFld.ParseValue(st.String()) + case utils.ANSWER_TIME: + at, _ := fsev.GetAnswerTime("") + return rsrFld.ParseValue(at.String()) + case utils.USAGE: + dur, _ := fsev.GetDuration("") + return rsrFld.ParseValue(strconv.FormatInt(dur.Nanoseconds(), 10)) + case utils.MEDI_RUNID: + return rsrFld.ParseValue(utils.DEFAULT_RUNID) + case utils.COST: + return rsrFld.ParseValue(strconv.FormatFloat(-1, 'f', -1, 64)) // Recommended to use FormatCost + default: + return rsrFld.ParseValue(fsev[rsrFld.Id]) + } +} diff --git a/sessionmanager/fsevent_test.go b/sessionmanager/fsevent_test.go index 961b7aeef..04ceab740 100644 --- a/sessionmanager/fsevent_test.go +++ b/sessionmanager/fsevent_test.go @@ -26,138 +26,7 @@ import ( "github.com/cgrates/cgrates/utils" ) -func TestEventCreation(t *testing.T) { - body := `Event-Name: RE_SCHEDULE -Core-UUID: 792e181c-b6e6-499c-82a1-52a778e7d82d -FreeSWITCH-Hostname: h1.ip-switch.net -FreeSWITCH-Switchname: h1.ip-switch.net -FreeSWITCH-IPv4: 88.198.12.156 -FreeSWITCH-IPv6: %3A%3A1 -Event-Date-Local: 2012-10-05%2013%3A41%3A38 -Event-Date-GMT: Fri,%2005%20Oct%202012%2011%3A41%3A38%20GMT -Event-Date-Timestamp: 1349437298012866 -Event-Calling-File: switch_scheduler.c -Event-Calling-Function: switch_scheduler_execute -Event-Calling-Line-Number: 65 -Event-Sequence: 34263 -Task-ID: 2 -Task-Desc: heartbeat -Task-Group: core -Task-Runtime: 1349437318` - ev := new(FSEvent).New(body) - if ev.GetName() != "RE_SCHEDULE" { - t.Error("Event not parsed correctly: ", ev) - } - l := len(ev.(FSEvent)) - if l != 17 { - t.Error("Incorrect number of event fields: ", l) - } -} - -// Detects if any of the parsers do not return static values -func TestEventParseStatic(t *testing.T) { - ev := new(FSEvent).New("") - setupTime, _ := ev.GetSetupTime("^2013-12-07 08:42:24") - answerTime, _ := ev.GetAnswerTime("^2013-12-07 08:42:24") - dur, _ := ev.GetDuration("^60s") - if ev.GetReqType("^test") != "test" || - ev.GetDirection("^test") != "test" || - ev.GetTenant("^test") != "test" || - ev.GetCategory("^test") != "test" || - ev.GetAccount("^test") != "test" || - ev.GetSubject("^test") != "test" || - ev.GetDestination("^test") != "test" || - setupTime != time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC) || - answerTime != time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC) || - dur != time.Duration(60)*time.Second { - t.Error("Values out of static not matching", - ev.GetReqType("^test") != "test", - ev.GetDirection("^test") != "test", - ev.GetTenant("^test") != "test", - ev.GetCategory("^test") != "test", - ev.GetAccount("^test") != "test", - ev.GetSubject("^test") != "test", - ev.GetDestination("^test") != "test", - setupTime != time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), - answerTime != time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), - dur != time.Duration(60)*time.Second) - } -} - -// Test here if the answer is selected out of headers we specify, even if not default defined -func TestEventSelectiveHeaders(t *testing.T) { - body := `Event-Name: RE_SCHEDULE -Core-UUID: 792e181c-b6e6-499c-82a1-52a778e7d82d -FreeSWITCH-Hostname: h1.ip-switch.net -FreeSWITCH-Switchname: h1.ip-switch.net -FreeSWITCH-IPv4: 88.198.12.156 -FreeSWITCH-IPv6: %3A%3A1 -Event-Date-Local: 2012-10-05%2013%3A41%3A38 -Event-Date-GMT: Fri,%2005%20Oct%202012%2011%3A41%3A38%20GMT -Event-Date-Timestamp: 1349437298012866 -Event-Calling-File: switch_scheduler.c -Event-Calling-Function: switch_scheduler_execute -Event-Calling-Line-Number: 65 -Event-Sequence: 34263 -Task-ID: 2 -Task-Desc: heartbeat -Task-Group: core -Task-Runtime: 1349437318` - cfg, _ = config.NewDefaultCGRConfig() - config.SetCgrConfig(cfg) - ev := new(FSEvent).New(body) - setupTime, _ := ev.GetSetupTime("Event-Date-Local") - answerTime, _ := ev.GetAnswerTime("Event-Date-Local") - dur, _ := ev.GetDuration("Event-Calling-Line-Number") - if ev.GetReqType("FreeSWITCH-Hostname") != "h1.ip-switch.net" || - ev.GetDirection("FreeSWITCH-Hostname") != "*out" || - ev.GetTenant("FreeSWITCH-Hostname") != "h1.ip-switch.net" || - ev.GetCategory("FreeSWITCH-Hostname") != "h1.ip-switch.net" || - ev.GetAccount("FreeSWITCH-Hostname") != "h1.ip-switch.net" || - ev.GetSubject("FreeSWITCH-Hostname") != "h1.ip-switch.net" || - ev.GetDestination("FreeSWITCH-Hostname") != "h1.ip-switch.net" || - setupTime != time.Date(2012, 10, 5, 13, 41, 38, 0, time.UTC) || - answerTime != time.Date(2012, 10, 5, 13, 41, 38, 0, time.UTC) || - dur != time.Duration(65)*time.Second { - t.Error("Values out of static not matching", - ev.GetReqType("FreeSWITCH-Hostname") != "h1.ip-switch.net", - ev.GetDirection("FreeSWITCH-Hostname") != "*out", - ev.GetTenant("FreeSWITCH-Hostname") != "h1.ip-switch.net", - ev.GetCategory("FreeSWITCH-Hostname") != "h1.ip-switch.net", - ev.GetAccount("FreeSWITCH-Hostname") != "h1.ip-switch.net", - ev.GetSubject("FreeSWITCH-Hostname") != "h1.ip-switch.net", - ev.GetDestination("FreeSWITCH-Hostname") != "h1.ip-switch.net", - setupTime != time.Date(2012, 10, 5, 13, 41, 38, 0, time.UTC), - answerTime != time.Date(2012, 10, 5, 13, 41, 38, 0, time.UTC), - dur != time.Duration(65)*time.Second) - } -} - -func TestDDazEmptyTime(t *testing.T) { - body := `Event-Name: RE_SCHEDULE -Core-UUID: 792e181c-b6e6-499c-82a1-52a778e7d82d -FreeSWITCH-Hostname: h1.ip-switch.net -FreeSWITCH-Switchname: h1.ip-switch.net -FreeSWITCH-IPv4: 88.198.12.156 -Caller-Channel-Created-Time: 0 -Caller-Channel-Answered-Time -Task-Runtime: 1349437318` - var nilTime time.Time - ev := new(FSEvent).New(body) - if setupTime, err := ev.GetSetupTime(""); err != nil { - t.Error("Error when parsing empty setupTime") - } else if setupTime != nilTime { - t.Error("Expecting nil time, got: ", setupTime) - } - if answerTime, err := ev.GetAnswerTime(""); err != nil { - t.Error("Error when parsing empty setupTime") - } else if answerTime != nilTime { - t.Error("Expecting nil time, got: ", answerTime) - } -} - -func TestParseFsHangup(t *testing.T) { - hangupEv := `Event-Name: CHANNEL_HANGUP_COMPLETE +var hangupEv string = `Event-Name: CHANNEL_HANGUP_COMPLETE Core-UUID: bb890f9e-0aae-476d-8292-91b434eb4f73 FreeSWITCH-Hostname: iPBXDev FreeSWITCH-Switchname: iPBXDev @@ -469,6 +338,138 @@ variable_rtp_audio_out_cng_packet_count: 0 variable_rtp_audio_rtcp_packet_count: 0 variable_rtp_audio_rtcp_octet_count: 0 ` + +func TestEventCreation(t *testing.T) { + body := `Event-Name: RE_SCHEDULE +Core-UUID: 792e181c-b6e6-499c-82a1-52a778e7d82d +FreeSWITCH-Hostname: h1.ip-switch.net +FreeSWITCH-Switchname: h1.ip-switch.net +FreeSWITCH-IPv4: 88.198.12.156 +FreeSWITCH-IPv6: %3A%3A1 +Event-Date-Local: 2012-10-05%2013%3A41%3A38 +Event-Date-GMT: Fri,%2005%20Oct%202012%2011%3A41%3A38%20GMT +Event-Date-Timestamp: 1349437298012866 +Event-Calling-File: switch_scheduler.c +Event-Calling-Function: switch_scheduler_execute +Event-Calling-Line-Number: 65 +Event-Sequence: 34263 +Task-ID: 2 +Task-Desc: heartbeat +Task-Group: core +Task-Runtime: 1349437318` + ev := new(FSEvent).New(body) + if ev.GetName() != "RE_SCHEDULE" { + t.Error("Event not parsed correctly: ", ev) + } + l := len(ev.(FSEvent)) + if l != 17 { + t.Error("Incorrect number of event fields: ", l) + } +} + +// Detects if any of the parsers do not return static values +func TestEventParseStatic(t *testing.T) { + ev := new(FSEvent).New("") + setupTime, _ := ev.GetSetupTime("^2013-12-07 08:42:24") + answerTime, _ := ev.GetAnswerTime("^2013-12-07 08:42:24") + dur, _ := ev.GetDuration("^60s") + if ev.GetReqType("^test") != "test" || + ev.GetDirection("^test") != "test" || + ev.GetTenant("^test") != "test" || + ev.GetCategory("^test") != "test" || + ev.GetAccount("^test") != "test" || + ev.GetSubject("^test") != "test" || + ev.GetDestination("^test") != "test" || + setupTime != time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC) || + answerTime != time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC) || + dur != time.Duration(60)*time.Second { + t.Error("Values out of static not matching", + ev.GetReqType("^test") != "test", + ev.GetDirection("^test") != "test", + ev.GetTenant("^test") != "test", + ev.GetCategory("^test") != "test", + ev.GetAccount("^test") != "test", + ev.GetSubject("^test") != "test", + ev.GetDestination("^test") != "test", + setupTime != time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), + answerTime != time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), + dur != time.Duration(60)*time.Second) + } +} + +// Test here if the answer is selected out of headers we specify, even if not default defined +func TestEventSelectiveHeaders(t *testing.T) { + body := `Event-Name: RE_SCHEDULE +Core-UUID: 792e181c-b6e6-499c-82a1-52a778e7d82d +FreeSWITCH-Hostname: h1.ip-switch.net +FreeSWITCH-Switchname: h1.ip-switch.net +FreeSWITCH-IPv4: 88.198.12.156 +FreeSWITCH-IPv6: %3A%3A1 +Event-Date-Local: 2012-10-05%2013%3A41%3A38 +Event-Date-GMT: Fri,%2005%20Oct%202012%2011%3A41%3A38%20GMT +Event-Date-Timestamp: 1349437298012866 +Event-Calling-File: switch_scheduler.c +Event-Calling-Function: switch_scheduler_execute +Event-Calling-Line-Number: 65 +Event-Sequence: 34263 +Task-ID: 2 +Task-Desc: heartbeat +Task-Group: core +Task-Runtime: 1349437318` + cfg, _ = config.NewDefaultCGRConfig() + config.SetCgrConfig(cfg) + ev := new(FSEvent).New(body) + setupTime, _ := ev.GetSetupTime("Event-Date-Local") + answerTime, _ := ev.GetAnswerTime("Event-Date-Local") + dur, _ := ev.GetDuration("Event-Calling-Line-Number") + if ev.GetReqType("FreeSWITCH-Hostname") != "h1.ip-switch.net" || + ev.GetDirection("FreeSWITCH-Hostname") != "*out" || + ev.GetTenant("FreeSWITCH-Hostname") != "h1.ip-switch.net" || + ev.GetCategory("FreeSWITCH-Hostname") != "h1.ip-switch.net" || + ev.GetAccount("FreeSWITCH-Hostname") != "h1.ip-switch.net" || + ev.GetSubject("FreeSWITCH-Hostname") != "h1.ip-switch.net" || + ev.GetDestination("FreeSWITCH-Hostname") != "h1.ip-switch.net" || + setupTime != time.Date(2012, 10, 5, 13, 41, 38, 0, time.UTC) || + answerTime != time.Date(2012, 10, 5, 13, 41, 38, 0, time.UTC) || + dur != time.Duration(65)*time.Second { + t.Error("Values out of static not matching", + ev.GetReqType("FreeSWITCH-Hostname") != "h1.ip-switch.net", + ev.GetDirection("FreeSWITCH-Hostname") != "*out", + ev.GetTenant("FreeSWITCH-Hostname") != "h1.ip-switch.net", + ev.GetCategory("FreeSWITCH-Hostname") != "h1.ip-switch.net", + ev.GetAccount("FreeSWITCH-Hostname") != "h1.ip-switch.net", + ev.GetSubject("FreeSWITCH-Hostname") != "h1.ip-switch.net", + ev.GetDestination("FreeSWITCH-Hostname") != "h1.ip-switch.net", + setupTime != time.Date(2012, 10, 5, 13, 41, 38, 0, time.UTC), + answerTime != time.Date(2012, 10, 5, 13, 41, 38, 0, time.UTC), + dur != time.Duration(65)*time.Second) + } +} + +func TestDDazEmptyTime(t *testing.T) { + body := `Event-Name: RE_SCHEDULE +Core-UUID: 792e181c-b6e6-499c-82a1-52a778e7d82d +FreeSWITCH-Hostname: h1.ip-switch.net +FreeSWITCH-Switchname: h1.ip-switch.net +FreeSWITCH-IPv4: 88.198.12.156 +Caller-Channel-Created-Time: 0 +Caller-Channel-Answered-Time +Task-Runtime: 1349437318` + var nilTime time.Time + ev := new(FSEvent).New(body) + if setupTime, err := ev.GetSetupTime(""); err != nil { + t.Error("Error when parsing empty setupTime") + } else if setupTime != nilTime { + t.Error("Expecting nil time, got: ", setupTime) + } + if answerTime, err := ev.GetAnswerTime(""); err != nil { + t.Error("Error when parsing empty setupTime") + } else if answerTime != nilTime { + t.Error("Expecting nil time, got: ", answerTime) + } +} + +func TestParseFsHangup(t *testing.T) { cfg, _ = config.NewDefaultCGRConfig() config.SetCgrConfig(cfg) ev := new(FSEvent).New(hangupEv) @@ -498,3 +499,63 @@ variable_rtp_audio_rtcp_octet_count: 0 dur != time.Duration(5)*time.Second) } } + +func TestParseEventValue(t *testing.T) { + cfg, _ = config.NewDefaultCGRConfig() + config.SetCgrConfig(cfg) + ev := new(FSEvent).New(hangupEv) + if cgrid := ev.ParseEventValue(&utils.RSRField{Id: utils.CGRID}); cgrid != "8b1ca78a9bbaa42c811e60b974188197c425dbe7" { + t.Error("Unexpected cgrid parsed", cgrid) + } + if tor := ev.ParseEventValue(&utils.RSRField{Id: utils.TOR}); tor != utils.VOICE { + t.Error("Unexpected tor parsed", tor) + } + if accid := ev.ParseEventValue(&utils.RSRField{Id: utils.ACCID}); accid != "37e9b766-5256-4e4b-b1ed-3767b930fec8" { + t.Error("Unexpected result parsed", accid) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.CDRHOST}); parsed != "10.0.2.15" { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.CDRSOURCE}); parsed != "FS_EVENT" { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.REQTYPE}); parsed != utils.PSEUDOPREPAID { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.DIRECTION}); parsed != utils.OUT { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.TENANT}); parsed != "cgrates.org" { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.CATEGORY}); parsed != "call" { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.ACCOUNT}); parsed != "1003" { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.SUBJECT}); parsed != "1003" { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.DESTINATION}); parsed != "1002" { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.SETUP_TIME}); parsed != "2014-04-25 18:08:27 +0200 CEST" { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.ANSWER_TIME}); parsed != "2014-04-25 18:08:40 +0200 CEST" { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.USAGE}); parsed != "5000000000" { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.MEDI_RUNID}); parsed != utils.DEFAULT_RUNID { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.COST}); parsed != "-1" { + t.Error("Unexpected result parsed", parsed) + } + if parsed := ev.ParseEventValue(&utils.RSRField{Id: "Hangup-Cause"}); parsed != "NORMAL_CLEARING" { + t.Error("Unexpected result parsed", parsed) + } +} diff --git a/sessionmanager/fssessionmanager.go b/sessionmanager/fssessionmanager.go index bbe46e0d5..5a5c8685b 100644 --- a/sessionmanager/fssessionmanager.go +++ b/sessionmanager/fssessionmanager.go @@ -161,6 +161,10 @@ func (sm *FSSessionManager) OnChannelPark(ev Event) { } dcs, _ = dcs.AppendDefaultRun() for _, dc := range dcs { + dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) + if dcRunFilter != nil && ev.ParseEventValue(&utils.RSRField{Id: dcRunFilter.Id}) != ev.ParseEventValue(dcRunFilter) { + engine.Logger.Info(fmt.Sprintf("Ignoring DerivedCharger with id %s due to non matching filter", dc.RunId)) + } startTime, err := ev.GetAnswerTime(PARK_TIME) if err != nil { engine.Logger.Err("Error parsing answer event start time, using time.Now!") diff --git a/utils/derivedchargers.go b/utils/derivedchargers.go index 2795ddf40..296d05d15 100644 --- a/utils/derivedchargers.go +++ b/utils/derivedchargers.go @@ -24,67 +24,75 @@ import ( ) // Wraps regexp compiling in case of rsr fields -func NewDerivedCharger(runId, reqTypeFld, dirFld, tenantFld, catFld, acntFld, subjFld, dstFld, sTimeFld, aTimeFld, durFld string) (dc *DerivedCharger, err error) { +func NewDerivedCharger(runId, runFilter, reqTypeFld, dirFld, tenantFld, catFld, acntFld, subjFld, dstFld, sTimeFld, aTimeFld, durFld string) (dc *DerivedCharger, err error) { if len(runId) == 0 { return nil, errors.New("Empty run id field") } dc = &DerivedCharger{RunId: runId} + dc.RunFilter = runFilter + if strings.HasPrefix(dc.RunFilter, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { + if dc.rsrRunFilter, err = NewRSRField(dc.RunFilter); err != nil { + return nil, err + } else if len(dc.rsrRunFilter.Id) == 0 { + return nil, errors.New("Empty filter header.") + } + } dc.ReqTypeField = reqTypeFld - if strings.HasPrefix(dc.ReqTypeField, REGEXP_PREFIX) { + if strings.HasPrefix(dc.ReqTypeField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { if dc.rsrReqTypeField, err = NewRSRField(dc.ReqTypeField); err != nil { return nil, err } } dc.DirectionField = dirFld - if strings.HasPrefix(dc.DirectionField, REGEXP_PREFIX) { + if strings.HasPrefix(dc.DirectionField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { if dc.rsrDirectionField, err = NewRSRField(dc.DirectionField); err != nil { return nil, err } } dc.TenantField = tenantFld - if strings.HasPrefix(dc.TenantField, REGEXP_PREFIX) { + if strings.HasPrefix(dc.TenantField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { if dc.rsrTenantField, err = NewRSRField(dc.TenantField); err != nil { return nil, err } } dc.CategoryField = catFld - if strings.HasPrefix(dc.CategoryField, REGEXP_PREFIX) { + if strings.HasPrefix(dc.CategoryField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { if dc.rsrCategoryField, err = NewRSRField(dc.CategoryField); err != nil { return nil, err } } dc.AccountField = acntFld - if strings.HasPrefix(dc.AccountField, REGEXP_PREFIX) { + if strings.HasPrefix(dc.AccountField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { if dc.rsrAccountField, err = NewRSRField(dc.AccountField); err != nil { return nil, err } } dc.SubjectField = subjFld - if strings.HasPrefix(dc.SubjectField, REGEXP_PREFIX) { + if strings.HasPrefix(dc.SubjectField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { if dc.rsrSubjectField, err = NewRSRField(dc.SubjectField); err != nil { return nil, err } } dc.DestinationField = dstFld - if strings.HasPrefix(dc.DestinationField, REGEXP_PREFIX) { + if strings.HasPrefix(dc.DestinationField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { if dc.rsrDestinationField, err = NewRSRField(dc.DestinationField); err != nil { return nil, err } } dc.SetupTimeField = sTimeFld - if strings.HasPrefix(dc.SetupTimeField, REGEXP_PREFIX) { + if strings.HasPrefix(dc.SetupTimeField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { if dc.rsrSetupTimeField, err = NewRSRField(dc.SetupTimeField); err != nil { return nil, err } } dc.AnswerTimeField = aTimeFld - if strings.HasPrefix(dc.AnswerTimeField, REGEXP_PREFIX) { + if strings.HasPrefix(dc.AnswerTimeField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { if dc.rsrAnswerTimeField, err = NewRSRField(dc.AnswerTimeField); err != nil { return nil, err } } dc.UsageField = durFld - if strings.HasPrefix(dc.UsageField, REGEXP_PREFIX) { + if strings.HasPrefix(dc.UsageField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { if dc.rsrUsageField, err = NewRSRField(dc.UsageField); err != nil { return nil, err } @@ -94,6 +102,7 @@ func NewDerivedCharger(runId, reqTypeFld, dirFld, tenantFld, catFld, acntFld, su type DerivedCharger struct { RunId string // Unique runId in the chain + RunFilter string // Only run the charger if the filter matches ReqTypeField string // Field containing request type info, number in case of csv source, '^' as prefix in case of static values DirectionField string // Field containing direction info TenantField string // Field containing tenant info @@ -104,7 +113,8 @@ type DerivedCharger struct { SetupTimeField string // Field containing setup time information AnswerTimeField string // Field containing answer time information UsageField string // Field containing usage information - rsrReqTypeField *RSRField // Storage for compiled Regexp in case of RSRFields + rsrRunFilter *RSRField // Storage for compiled Regexp in case of RSRFields + rsrReqTypeField *RSRField rsrDirectionField *RSRField rsrTenantField *RSRField rsrCategoryField *RSRField @@ -136,7 +146,7 @@ func (dcs DerivedChargers) Append(dc *DerivedCharger) (DerivedChargers, error) { } func (dcs DerivedChargers) AppendDefaultRun() (DerivedChargers, error) { - dcDf, _ := NewDerivedCharger(DEFAULT_RUNID, META_DEFAULT, META_DEFAULT, META_DEFAULT, META_DEFAULT, META_DEFAULT, + dcDf, _ := NewDerivedCharger(DEFAULT_RUNID, "", META_DEFAULT, META_DEFAULT, META_DEFAULT, META_DEFAULT, META_DEFAULT, META_DEFAULT, META_DEFAULT, META_DEFAULT, META_DEFAULT, META_DEFAULT) return append(dcs, dcDf), nil } diff --git a/utils/derivedchargers_test.go b/utils/derivedchargers_test.go index 425cfb54d..52e2f1566 100644 --- a/utils/derivedchargers_test.go +++ b/utils/derivedchargers_test.go @@ -47,6 +47,7 @@ func TestAppendDerivedChargers(t *testing.T) { func TestNewDerivedCharger(t *testing.T) { edc1 := &DerivedCharger{ RunId: "test1", + RunFilter: "", ReqTypeField: "reqtype1", DirectionField: "direction1", TenantField: "tenant1", @@ -58,7 +59,7 @@ func TestNewDerivedCharger(t *testing.T) { AnswerTimeField: "answertime1", UsageField: "duration1", } - if dc1, err := NewDerivedCharger("test1", "reqtype1", "direction1", "tenant1", "tor1", "account1", "subject1", "destination1", + if dc1, err := NewDerivedCharger("test1", "", "reqtype1", "direction1", "tenant1", "tor1", "account1", "subject1", "destination1", "setuptime1", "answertime1", "duration1"); err != nil { t.Error("Unexpected error", err.Error) } else if !reflect.DeepEqual(edc1, dc1) { @@ -66,6 +67,7 @@ func TestNewDerivedCharger(t *testing.T) { } edc2 := &DerivedCharger{ RunId: "test2", + RunFilter: "^cdr_source/tdm_cdrs", ReqTypeField: "~reqtype2:s/sip:(.+)/$1/", DirectionField: "~direction2:s/sip:(.+)/$1/", TenantField: "~tenant2:s/sip:(.+)/$1/", @@ -77,6 +79,7 @@ func TestNewDerivedCharger(t *testing.T) { AnswerTimeField: "~answertime2:s/sip:(.+)/$1/", UsageField: "~duration2:s/sip:(.+)/$1/", } + edc2.rsrRunFilter, _ = NewRSRField("^cdr_source/tdm_cdrs") edc2.rsrReqTypeField, _ = NewRSRField("~reqtype2:s/sip:(.+)/$1/") edc2.rsrDirectionField, _ = NewRSRField("~direction2:s/sip:(.+)/$1/") edc2.rsrTenantField, _ = NewRSRField("~tenant2:s/sip:(.+)/$1/") @@ -88,6 +91,7 @@ func TestNewDerivedCharger(t *testing.T) { edc2.rsrAnswerTimeField, _ = NewRSRField("~answertime2:s/sip:(.+)/$1/") edc2.rsrUsageField, _ = NewRSRField("~duration2:s/sip:(.+)/$1/") if dc2, err := NewDerivedCharger("test2", + "^cdr_source/tdm_cdrs", "~reqtype2:s/sip:(.+)/$1/", "~direction2:s/sip:(.+)/$1/", "~tenant2:s/sip:(.+)/$1/", @@ -112,7 +116,7 @@ func TestDerivedChargersKey(t *testing.T) { func TestAppendDefaultRun(t *testing.T) { var dc1 DerivedChargers - dcDf := &DerivedCharger{RunId: DEFAULT_RUNID, ReqTypeField: META_DEFAULT, DirectionField: META_DEFAULT, + dcDf := &DerivedCharger{RunId: DEFAULT_RUNID, RunFilter: "", ReqTypeField: META_DEFAULT, DirectionField: META_DEFAULT, TenantField: META_DEFAULT, CategoryField: META_DEFAULT, AccountField: META_DEFAULT, SubjectField: META_DEFAULT, DestinationField: META_DEFAULT, SetupTimeField: META_DEFAULT, AnswerTimeField: META_DEFAULT, UsageField: META_DEFAULT} eDc1 := DerivedChargers{dcDf} @@ -120,7 +124,7 @@ func TestAppendDefaultRun(t *testing.T) { t.Error("Unexpected result.") } dc2 := DerivedChargers{ - &DerivedCharger{RunId: "extra1", ReqTypeField: "^prepaid", DirectionField: "*default", TenantField: "*default", CategoryField: "*default", + &DerivedCharger{RunId: "extra1", RunFilter: "", ReqTypeField: "reqtype2", DirectionField: "*default", TenantField: "*default", CategoryField: "*default", AccountField: "rif", SubjectField: "rif", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"}, &DerivedCharger{RunId: "extra2", ReqTypeField: "*default", DirectionField: "*default", TenantField: "*default", CategoryField: "*default", AccountField: "ivo", SubjectField: "ivo", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"}, From 31b421c7559469af38d31af8ad0b4030ef0678ce Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 7 Jul 2014 11:34:23 +0200 Subject: [PATCH 05/25] Loader for derived charging filters, using derived chargers filters within mediator and session manager, filter implementation inside StoredCdr --- .../mysql/create_tariffplan_tables.sql | 3 +- .../prepaid1centpsec/DerivedChargers.csv | 9 ++-- .../cgrates/tariffplans/DerivedChargers.csv | 4 +- engine/loader_csv.go | 42 ++++++++++--------- engine/loader_csv_test.go | 13 +++--- engine/loader_helpers.go | 4 +- engine/loader_helpers_test.go | 23 +++++----- mediator/mediator.go | 4 +- sessionmanager/fssessionmanager.go | 2 +- utils/consts.go | 2 +- utils/storedcdr.go | 11 +++++ utils/storedcdr_test.go | 26 ++++++++++++ 12 files changed, 93 insertions(+), 50 deletions(-) diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index 9557e7675..9aa46464d 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -246,7 +246,8 @@ CREATE TABLE tp_derived_chargers ( `category` varchar(16) NOT NULL, `account` varchar(24) NOT NULL, `subject` varchar(64) NOT NULL, - `runid_field` varchar(24) NOT NULL, + `run_id` varchar(24) NOT NULL, + `run_filter` varchar(24) NOT NULL, `reqtype_field` varchar(24) NOT NULL, `direction_field` varchar(24) NOT NULL, `tenant_field` varchar(24) NOT NULL, diff --git a/data/tariffplans/prepaid1centpsec/DerivedChargers.csv b/data/tariffplans/prepaid1centpsec/DerivedChargers.csv index 4b02d8f0e..7246f65fe 100644 --- a/data/tariffplans/prepaid1centpsec/DerivedChargers.csv +++ b/data/tariffplans/prepaid1centpsec/DerivedChargers.csv @@ -1,4 +1,5 @@ -#Direction,Tenant,Tor,Account,Subject,RunId,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,DurationField -*out,cgrates.org,call,dan,dan,extra1,^prepaid,,,,^rif,^rif,,,,^1s -*out,cgrates.org,call,dan,dan,extra2,,,,,^ivo,^ivo,,,, -*out,cgrates.org,call,dan,*any,extra1,,,,,^rif2,^rif2,,,, +#Direction,Tenant,Tor,Account,Subject,RunId,RunFilter,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,DurationField +*out,cgrates.org,call,dan,dan,extra1,,^prepaid,,,,^rif,^rif,,,,^1s +*out,cgrates.org,call,dan,dan,extra2,,,,,,^ivo,^ivo,,,, +*out,cgrates.org,call,dan,dan,extra3,~filterhdr1:s/(.+)/special_run3/,,,,,^runusr3,^runusr3,,,, +*out,cgrates.org,call,dan,*any,extra1,,,,,,^rif2,^rif2,,,, diff --git a/data/tutorials/fs_json/cgrates/tariffplans/DerivedChargers.csv b/data/tutorials/fs_json/cgrates/tariffplans/DerivedChargers.csv index 5a243dffd..aed05a80d 100644 --- a/data/tutorials/fs_json/cgrates/tariffplans/DerivedChargers.csv +++ b/data/tutorials/fs_json/cgrates/tariffplans/DerivedChargers.csv @@ -1,2 +1,2 @@ -#Direction,Tenant,Category,Account,Subject,RunId,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,UsageField -*out,cgrates.org,call,1001,1001,fs_json_run,^rated,*default,*default,*default,*default,^1002,*default,*default,*default,*default +#Direction,Tenant,Category,Account,Subject,RunId,RunFilter,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,UsageField +*out,cgrates.org,call,1001,1001,fs_json_run,,^rated,*default,*default,*default,*default,^1002,*default,*default,*default,*default diff --git a/engine/loader_csv.go b/engine/loader_csv.go index c44d83a58..47f2ab28f 100644 --- a/engine/loader_csv.go +++ b/engine/loader_csv.go @@ -842,16 +842,17 @@ func (csvr *CSVReader) LoadDerivedChargers() (err error) { if found { if csvr.derivedChargers[tag], err = csvr.derivedChargers[tag].Append(&utils.DerivedCharger{ RunId: ValueOrDefault(record[5], "*default"), - ReqTypeField: ValueOrDefault(record[6], "*default"), - DirectionField: ValueOrDefault(record[7], "*default"), - TenantField: ValueOrDefault(record[8], "*default"), - CategoryField: ValueOrDefault(record[9], "*default"), - AccountField: ValueOrDefault(record[10], "*default"), - SubjectField: ValueOrDefault(record[11], "*default"), - DestinationField: ValueOrDefault(record[12], "*default"), - SetupTimeField: ValueOrDefault(record[13], "*default"), - AnswerTimeField: ValueOrDefault(record[14], "*default"), - UsageField: ValueOrDefault(record[15], "*default"), + RunFilter: record[6], + ReqTypeField: ValueOrDefault(record[7], "*default"), + DirectionField: ValueOrDefault(record[8], "*default"), + TenantField: ValueOrDefault(record[9], "*default"), + CategoryField: ValueOrDefault(record[10], "*default"), + AccountField: ValueOrDefault(record[11], "*default"), + SubjectField: ValueOrDefault(record[12], "*default"), + DestinationField: ValueOrDefault(record[13], "*default"), + SetupTimeField: ValueOrDefault(record[14], "*default"), + AnswerTimeField: ValueOrDefault(record[15], "*default"), + UsageField: ValueOrDefault(record[16], "*default"), }); err != nil { return err } @@ -861,16 +862,17 @@ func (csvr *CSVReader) LoadDerivedChargers() (err error) { } csvr.derivedChargers[tag] = utils.DerivedChargers{&utils.DerivedCharger{ RunId: ValueOrDefault(record[5], "*default"), - ReqTypeField: ValueOrDefault(record[6], "*default"), - DirectionField: ValueOrDefault(record[7], "*default"), - TenantField: ValueOrDefault(record[8], "*default"), - CategoryField: ValueOrDefault(record[9], "*default"), - AccountField: ValueOrDefault(record[10], "*default"), - SubjectField: ValueOrDefault(record[11], "*default"), - DestinationField: ValueOrDefault(record[12], "*default"), - SetupTimeField: ValueOrDefault(record[13], "*default"), - AnswerTimeField: ValueOrDefault(record[14], "*default"), - UsageField: ValueOrDefault(record[15], "*default"), + RunFilter: record[6], + ReqTypeField: ValueOrDefault(record[7], "*default"), + DirectionField: ValueOrDefault(record[8], "*default"), + TenantField: ValueOrDefault(record[9], "*default"), + CategoryField: ValueOrDefault(record[10], "*default"), + AccountField: ValueOrDefault(record[11], "*default"), + SubjectField: ValueOrDefault(record[12], "*default"), + DestinationField: ValueOrDefault(record[13], "*default"), + SetupTimeField: ValueOrDefault(record[14], "*default"), + AnswerTimeField: ValueOrDefault(record[15], "*default"), + UsageField: ValueOrDefault(record[16], "*default"), }} } } diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 63ea352dd..82e62011e 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -177,10 +177,10 @@ vdf,emptyY,*out,TOPUP_EMPTY_AT, ` derivedCharges = ` -#Direction,Tenant,Category,Account,Subject,RunId,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,UsageField -*out,cgrates.org,call,dan,dan,extra1,^prepaid,,,,rif,rif,,,, -*out,cgrates.org,call,dan,dan,extra2,,,,,ivo,ivo,,,, -*out,cgrates.org,call,dan,*any,extra1,,,,,rif2,rif2,,,, +#Direction,Tenant,Category,Account,Subject,RunId,RunFilter,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,UsageField +*out,cgrates.org,call,dan,dan,extra1,^filteredHeader1/filterValue1,^prepaid,,,,rif,rif,,,, +*out,cgrates.org,call,dan,dan,extra2,,,,,,ivo,ivo,,,, +*out,cgrates.org,call,dan,*any,extra1,,,,,,rif2,rif2,,,, ` ) @@ -932,8 +932,9 @@ func TestLoadDerivedChargers(t *testing.T) { t.Error("Failed to load derivedChargers: ", csvr.derivedChargers) } expCharger1 := utils.DerivedChargers{ - &utils.DerivedCharger{RunId: "extra1", ReqTypeField: "^prepaid", DirectionField: "*default", TenantField: "*default", CategoryField: "*default", - AccountField: "rif", SubjectField: "rif", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"}, + &utils.DerivedCharger{RunId: "extra1", RunFilter: "^filteredHeader1/filterValue1", ReqTypeField: "^prepaid", DirectionField: "*default", TenantField: "*default", + CategoryField: "*default", AccountField: "rif", SubjectField: "rif", DestinationField: "*default", + SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"}, &utils.DerivedCharger{RunId: "extra2", ReqTypeField: "*default", DirectionField: "*default", TenantField: "*default", CategoryField: "*default", AccountField: "ivo", SubjectField: "ivo", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"}, } diff --git a/engine/loader_helpers.go b/engine/loader_helpers.go index 3b3b2fe44..896c2a662 100644 --- a/engine/loader_helpers.go +++ b/engine/loader_helpers.go @@ -208,8 +208,8 @@ var FileValidators = map[string]*FileLineRegexValidator{ regexp.MustCompile(`(?:\w+\s*),(?:(\w+;?)+\s*),(?:\*out\s*),(?:\w+\s*),(?:\w+\s*)$`), "Tenant([0-9A-Za-z_]),Account([0-9A-Za-z_.]),Direction(*out),ActionTimingsTag([0-9A-Za-z_]),ActionTriggersTag([0-9A-Za-z_])"}, utils.DERIVED_CHARGERS_CSV: &FileLineRegexValidator{utils.DERIVED_CHARGERS_NRCOLS, - regexp.MustCompile(`^(?:\*out),(?:[0-9A-Za-z_\.]+\s*),(?:\w+\s*),(?:\w+\s*),(?:\*any\s*|\w+\s*),(?:\w+\s*),(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?$`), - "Direction(*out),Tenant[0-9A-Za-z_],Category([0-9A-Za-z_]),Account[0-9A-Za-z_],Subject([0-9A-Za-z_]|*any),RunId([^0-9A-Za-z_]),ReqTypeField([^0-9A-Za-z_]|*default),DirectionField([^0-9A-Za-z_]|*default),TenantField([^0-9A-Za-z_]|*default),TorField([^0-9A-Za-z_]|*default),AccountField([^0-9A-Za-z_]|*default),SubjectField([^0-9A-Za-z_]|*default),DestinationField([^0-9A-Za-z_]|*default),SetupTimeField([^0-9A-Za-z_]|*default),AnswerTimeField([^0-9A-Za-z_]|*default),DurationField([^0-9A-Za-z_]|*default)"}, + regexp.MustCompile(`^(?:\*out),(?:[0-9A-Za-z_\.]+\s*),(?:\w+\s*),(?:\w+\s*),(?:\*any\s*|\w+\s*),(?:\w+\s*),(?:[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?$`), + "Direction(*out),Tenant[0-9A-Za-z_],Category([0-9A-Za-z_]),Account[0-9A-Za-z_],Subject([0-9A-Za-z_]|*any),RunId([0-9A-Za-z_]),RunFilter([^~]*[0-9A-Za-z_/]),ReqTypeField([^~]*[0-9A-Za-z_/]|*default),DirectionField([^~]*[0-9A-Za-z_/]|*default),TenantField([^~]*[0-9A-Za-z_/]|*default),TorField([^~]*[0-9A-Za-z_/]|*default),AccountField([^~]*[0-9A-Za-z_/]|*default),SubjectField([^~]*[0-9A-Za-z_/]|*default),DestinationField([^~]*[0-9A-Za-z_/]|*default),SetupTimeField([^~]*[0-9A-Za-z_/]|*default),AnswerTimeField([^~]*[0-9A-Za-z_/]|*default),DurationField([^~]*[0-9A-Za-z_/]|*default)"}, } func NewTPCSVFileParser(dirPath, fileName string) (*TPCSVFileParser, error) { diff --git a/engine/loader_helpers_test.go b/engine/loader_helpers_test.go index b8dd0a0a9..57dde2d87 100644 --- a/engine/loader_helpers_test.go +++ b/engine/loader_helpers_test.go @@ -93,17 +93,18 @@ DUMMY,INVALID;DATA cgrates.org,1002;1006,*out,PACKAGE_10,STANDARD_TRIGGERS ` var derivedChargesSample = `#Direction,Tenant,Tor,Account,Subject,RunId,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,DurationField -*out,cgrates.org,call,dan,dan,extra1,^prepaid,,,,rif,rif,,,, -*out,cgrates.org,,dan,dan,extra1,^prepaid,,,,rif,rif,,,, -*in,cgrates.org,call,dan,dan,extra1,^prepaid,,,,rif,rif,,,, +*out,cgrates.org,call,dan,dan,extra1,^filteredHeader1/filterValue1,^prepaid,,,,rif,rif,,,, +*out,cgrates.org,,dan,dan,extra1,^filteredHeader1/filterValue1,^prepaid,,,,rif,rif,,,, +*in,cgrates.org,call,dan,dan,extra1,^filteredHeader1/filterValue1,^prepaid,,,,rif,rif,,,, DUMMY_DATA -*out,cgrates.org,call,dan,dan,extra2,,,,,ivo,ivo,,,, -*out,cgrates.org,call,dan,*any,extra1,,,,,rif2,rif2,,,, -*out,cgrates.org,call,dan,*any,*any,,,,,rif2,rif2,,,, -*out,cgrates.org,call,dan,*any,*default,*default,*default,*default,*default,rif2,rif2,*default,*default,*default,*default -*out,cgrates.org,call,dan,*any,test,^test,^test,^test,^test,^test,^test,^test,^test,^test,^test -*out,cgrates.org,call,dan,*any,run1,,,,,,,,,, -*out,cgrates.org,call,dan,*default,,,,,,,,,,, +*out,cgrates.org,call,dan,dan,extra2,,,,,,ivo,ivo,,,, +*out,cgrates.org,call,dan,*any,extra1,,,,,,rif2,rif2,,,, +*out,cgrates.org,call,dan,*any,*any,,,,,,rif2,rif2,,,, +*out,cgrates.org,call,dan,*any,*default,,*default,*default,*default,*default,rif2,rif2,*default,*default,*default,*default +*out,cgrates.org,call,dan,*any,test,^test,^test,^test,^test,^test,^test,^test,^test,^test,^test,^test +*out,cgrates.org,call,dan,*any,run1,,,,,,,,,,, +*out,cgrates.org,call,dan,*default,,,,,,,,,,,, +*out,cgrates.org,call,dan,dan,extra3,~filterhdr1:s/(.+)/special_run3/,,,,,^runusr3,^runusr3,,,, ` func TestTimingsValidator(t *testing.T) { @@ -386,7 +387,7 @@ func TestDerivedChargersValidator(t *testing.T) { if valid { t.Error("Validation passed for invalid line", string(ln)) } - case 2, 6, 7, 10, 11: + case 2, 6, 7, 10, 11, 13: if !valid { t.Error("Validation did not pass for valid line", string(ln)) } diff --git a/mediator/mediator.go b/mediator/mediator.go index 6dc813935..d0fccb4f2 100644 --- a/mediator/mediator.go +++ b/mediator/mediator.go @@ -121,8 +121,8 @@ func (self *Mediator) RateCdr(storedCdr *utils.StoredCdr) error { } for _, dc := range dcs { dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) - if dcRunFilter != nil && storedCdr.FieldAsString(&utils.RSRField{Id: dcRunFilter.Id}) != storedCdr.FieldAsString(dcRunFilter) { - engine.Logger.Info(fmt.Sprintf("Ignoring DerivedCharger with id %s due to non matching filter", dc.RunId)) + if !storedCdr.PassesFieldFilter(dcRunFilter) { + engine.Logger.Info(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) continue } dcReqTypeFld, _ := utils.NewRSRField(dc.ReqTypeField) diff --git a/sessionmanager/fssessionmanager.go b/sessionmanager/fssessionmanager.go index 5a5c8685b..161fbfeef 100644 --- a/sessionmanager/fssessionmanager.go +++ b/sessionmanager/fssessionmanager.go @@ -163,7 +163,7 @@ func (sm *FSSessionManager) OnChannelPark(ev Event) { for _, dc := range dcs { dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) if dcRunFilter != nil && ev.ParseEventValue(&utils.RSRField{Id: dcRunFilter.Id}) != ev.ParseEventValue(dcRunFilter) { - engine.Logger.Info(fmt.Sprintf("Ignoring DerivedCharger with id %s due to non matching filter", dc.RunId)) + engine.Logger.Info(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) } startTime, err := ev.GetAnswerTime(PARK_TIME) if err != nil { diff --git a/utils/consts.go b/utils/consts.go index 6bb86e304..f3dd508d6 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -61,7 +61,7 @@ const ( ACTION_PLANS_NRCOLS = 4 ACTION_TRIGGERS_NRCOLS = 9 ACCOUNT_ACTIONS_NRCOLS = 5 - DERIVED_CHARGERS_NRCOLS = 16 + DERIVED_CHARGERS_NRCOLS = 17 ROUNDING_UP = "*up" ROUNDING_MIDDLE = "*middle" ROUNDING_DOWN = "*down" diff --git a/utils/storedcdr.go b/utils/storedcdr.go index 2c2d03a5a..733b84d80 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -133,6 +133,17 @@ func (storedCdr *StoredCdr) FieldAsString(rsrFld *RSRField) string { } } +func (storedCdr *StoredCdr) PassesFieldFilter(fieldFilter *RSRField) bool { + if fieldFilter == nil { + return true + } + if storedCdr.FieldAsString(&RSRField{Id: fieldFilter.Id}) == storedCdr.FieldAsString(fieldFilter) && len(storedCdr.FieldAsString(fieldFilter)) != 0 { + // Field value must be non empty in order to declare it filtered, otherwise filter makes no sense + return true + } + return false +} + func (storedCdr *StoredCdr) AsStoredCdr() *StoredCdr { return storedCdr } diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index baf8018a3..6b5c9022a 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -78,6 +78,32 @@ func TestFieldAsString(t *testing.T) { } } +func TestPassesFieldFilter(t *testing.T) { + cdr := StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", + Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID, + Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + } + if !cdr.PassesFieldFilter(nil) { + t.Error("Not passing filter") + } + acntPrefxFltr, _ := NewRSRField(`~account:s/(.+)/1001/`) + if !cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Not passing filter") + } + torFltr, _ := NewRSRField(`^tor/*voice`) + if !cdr.PassesFieldFilter(torFltr) { + t.Error("Not passing filter") + } + torFltr, _ = NewRSRField(`^tor/*data`) + if cdr.PassesFieldFilter(torFltr) { + t.Error("Passing filter") + } + inexistentFieldFltr, _ := NewRSRField(`^fakefield/fakevalue`) + if cdr.PassesFieldFilter(inexistentFieldFltr) { + t.Error("Passing filter") + } +} + func TestUsageMultiply(t *testing.T) { cdr := StoredCdr{Usage: time.Duration(10) * time.Second} if cdr.UsageMultiply(1024.0, 0); cdr.Usage != time.Duration(10240)*time.Second { From e96251aba345b6befb0b1cb95d96bde270fb2d98 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 7 Jul 2014 17:48:59 +0200 Subject: [PATCH 06/25] RegexpSearchReplace rule has now Matched field to confirm matching, DerivedChargers has now Regexp support in filters --- cdrc/cdrc.go | 6 ++++-- cdrs/fscdr_test.go | 4 ++-- config/config_test.go | 2 +- config/helpers_test.go | 3 ++- utils/researchreplace.go | 3 +++ utils/researchreplace_test.go | 8 ++++---- utils/rsrfield.go | 13 +++++++++++++ utils/rsrfield_test.go | 31 +++++++++++++++++++++++-------- utils/storedcdr.go | 11 +++++++++-- utils/storedcdr_test.go | 22 ++++++++++++++++++++-- 10 files changed, 81 insertions(+), 22 deletions(-) diff --git a/cdrc/cdrc.go b/cdrc/cdrc.go index f485dc61f..c9b641092 100644 --- a/cdrc/cdrc.go +++ b/cdrc/cdrc.go @@ -196,7 +196,9 @@ func (self *Cdrc) processFile(filePath string) error { } csvReader := csv.NewReader(bufio.NewReader(file)) csvReader.Comma = self.csvSep + procRowNr := 0 for { + procRowNr += 1 record, err := csvReader.Read() if err != nil && err == io.EOF { break // End of file @@ -211,12 +213,12 @@ func (self *Cdrc) processFile(filePath string) error { } if self.cdrsAddress == utils.INTERNAL { if err := self.cdrServer.ProcessRawCdr(storedCdr); err != nil { - engine.Logger.Err(fmt.Sprintf(" Failed posting CDR, error: %s", err.Error())) + engine.Logger.Err(fmt.Sprintf(" Failed posting CDR, row: %d, error: %s", procRowNr, err.Error())) continue } } else { // CDRs listening on IP if _, err := self.httpClient.PostForm(fmt.Sprintf("http://%s/cgr", self.cdrsAddress), storedCdr.AsHttpForm()); err != nil { - engine.Logger.Err(fmt.Sprintf(" Failed posting CDR, error: %s", err.Error())) + engine.Logger.Err(fmt.Sprintf(" Failed posting CDR, row: %d, error: %s", procRowNr, err.Error())) continue } } diff --git a/cdrs/fscdr_test.go b/cdrs/fscdr_test.go index 4ac83b1b8..1c4ffe2fb 100644 --- a/cdrs/fscdr_test.go +++ b/cdrs/fscdr_test.go @@ -95,7 +95,7 @@ func TestSearchExtraFieldInSlice(t *testing.T) { func TestSearchReplaceInExtraFields(t *testing.T) { cfg, _ = config.NewDefaultCGRConfig() cfg.CDRSExtraFields = []*utils.RSRField{&utils.RSRField{Id: "read_codec"}, - &utils.RSRField{Id: "sip_user_agent", RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{regexp.MustCompile(`([A-Za-z]*).+`), "$1"}}}, + &utils.RSRField{Id: "sip_user_agent", RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`([A-Za-z]*).+`), ReplaceTemplate: "$1"}}}, &utils.RSRField{Id: "write_codec"}} fsCdr, _ := NewFSCdr(body) extraFields := fsCdr.getExtraFields() @@ -148,7 +148,7 @@ extra_fields = ~effective_caller_id_number:s/(\d+)/+$1/ if err != nil { t.Error("Could not parse the config", err.Error()) } else if !reflect.DeepEqual(cfg.CDRSExtraFields, []*utils.RSRField{&utils.RSRField{Id: "effective_caller_id_number", - RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{regexp.MustCompile(`(\d+)`), "+$1"}}}}) { + RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"}}}}) { t.Errorf("Unexpected value for config CdrsExtraFields: %v", cfg.CDRSExtraFields) } fsCdr, err := NewFSCdr(simpleJsonCdr) diff --git a/config/config_test.go b/config/config_test.go index dc90fd857..1c4eaab56 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -290,7 +290,7 @@ extra_fields = ~effective_caller_id_number:s/(\d+)/+$1/ if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil { t.Error("Could not parse the config", err.Error()) } else if !reflect.DeepEqual(cfg.CDRSExtraFields, []*utils.RSRField{&utils.RSRField{Id: "effective_caller_id_number", - RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{regexp.MustCompile(`(\d+)`), "+$1"}}}}) { + RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"}}}}) { t.Errorf("Unexpected value for config CdrsExtraFields: %v", cfg.CDRSExtraFields) } eFieldsCfg = []byte(`[cdrs] diff --git a/config/helpers_test.go b/config/helpers_test.go index 13aaeff36..31aa4747c 100644 --- a/config/helpers_test.go +++ b/config/helpers_test.go @@ -38,7 +38,8 @@ func TestConfigSlice(t *testing.T) { func TestParseRSRFields(t *testing.T) { fields := `host,~sip_redirected_to:s/sip:\+49(\d+)@/0$1/,destination` expectParsedFields := []*utils.RSRField{&utils.RSRField{Id: "host"}, - &utils.RSRField{Id: "sip_redirected_to", RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{regexp.MustCompile(`sip:\+49(\d+)@`), "0$1"}}}, + &utils.RSRField{Id: "sip_redirected_to", + RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), ReplaceTemplate: "0$1"}}}, &utils.RSRField{Id: "destination"}} if parsedFields, err := ParseRSRFields(fields); err != nil { t.Error("Unexpected error: ", err.Error()) diff --git a/utils/researchreplace.go b/utils/researchreplace.go index 8cb8e6ea6..66e226877 100644 --- a/utils/researchreplace.go +++ b/utils/researchreplace.go @@ -26,6 +26,7 @@ import ( type ReSearchReplace struct { SearchRegexp *regexp.Regexp ReplaceTemplate string + Matched bool } func (rsr *ReSearchReplace) Process(source string) string { @@ -36,6 +37,8 @@ func (rsr *ReSearchReplace) Process(source string) string { match := rsr.SearchRegexp.FindStringSubmatchIndex(source) if match == nil { return source // No match returns unaltered source, so we can play with national vs international dialing + } else { + rsr.Matched = true } res = rsr.SearchRegexp.ExpandString(res, rsr.ReplaceTemplate, source, match) return string(res) diff --git a/utils/researchreplace_test.go b/utils/researchreplace_test.go index 28c113ce8..44f5c52fd 100644 --- a/utils/researchreplace_test.go +++ b/utils/researchreplace_test.go @@ -24,7 +24,7 @@ import ( ) func TestProcessReSearchReplace(t *testing.T) { - rsr := &ReSearchReplace{regexp.MustCompile(`sip:\+49(\d+)@(\d*\.\d*\.\d*\.\d*)`), "0$1@$2"} + rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@(\d*\.\d*\.\d*\.\d*)`), ReplaceTemplate: "0$1@$2"} source := "" expectOut := "086517174963@127.0.0.1" if outStr := rsr.Process(source); outStr != expectOut { @@ -33,7 +33,7 @@ func TestProcessReSearchReplace(t *testing.T) { } func TestProcessReSearchReplace2(t *testing.T) { - rsr := &ReSearchReplace{regexp.MustCompile(`(\d+)`), "+$1"} + rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"} source := "4986517174963" expectOut := "+4986517174963" if outStr := rsr.Process(source); outStr != expectOut { @@ -42,7 +42,7 @@ func TestProcessReSearchReplace2(t *testing.T) { } func TestProcessReSearchReplace3(t *testing.T) { //"MatchedDestId":"CST_31800_DE080" - rsr := &ReSearchReplace{regexp.MustCompile(`"MatchedDestId":".+_(\w{5})"`), "$1"} + rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`"MatchedDestId":".+_(\w{5})"`), ReplaceTemplate: "$1"} source := `[{"TimeStart":"2014-04-15T22:17:57+02:00","TimeEnd":"2014-04-15T22:18:01+02:00","Cost":0,"RateInterval":{"Timing":{"Years":[],"Months":[],"MonthDays":[],"WeekDays":[],"StartTime":"00:00:00","EndTime":""},"Rating":{"ConnectFee":0,"Rates":[{"GroupIntervalStart":0,"Value":0,"RateIncrement":1000000000,"RateUnit":60000000000}],"RoundingMethod":"*middle","RoundingDecimals":4},"Weight":10},"CallDuration":4000000000,"Increments":null,"MatchedSubject":"*out:sip.test.cgrates.org:call:*any","MatchedPrefix":"+49800","MatchedDestId":"CST_31800_DE080"}]` expectOut := "DE080" if outStr := rsr.Process(source); outStr != expectOut { @@ -51,7 +51,7 @@ func TestProcessReSearchReplace3(t *testing.T) { //"MatchedDestId":"CST_31800_DE } func TestProcessReSearchReplace4(t *testing.T) { - rsr := &ReSearchReplace{regexp.MustCompile(`^\+49(\d+)`), "0$1"} + rsr := &ReSearchReplace{SearchRegexp: regexp.MustCompile(`^\+49(\d+)`), ReplaceTemplate: "0$1"} if outStr := rsr.Process("+4986517174963"); outStr != "086517174963" { t.Error("Unexpected output from SearchReplace: ", outStr) } diff --git a/utils/rsrfield.go b/utils/rsrfield.go index 4af340f7f..56731428c 100644 --- a/utils/rsrfield.go +++ b/utils/rsrfield.go @@ -82,3 +82,16 @@ func (rsrf *RSRField) ParseValue(value string) string { } return value } + +func (rsrf *RSRField) IsStatic() bool { + return len(rsrf.staticValue) != 0 +} + +func (rsrf *RSRField) RegexpMatched() bool { // Investigate whether we had a regexp match through the rules + for _, rsrule := range rsrf.RSRules { + if rsrule.Matched { + return true + } + } + return false +} diff --git a/utils/rsrfield_test.go b/utils/rsrfield_test.go index 7b078bb89..7b215f161 100644 --- a/utils/rsrfield_test.go +++ b/utils/rsrfield_test.go @@ -26,7 +26,8 @@ import ( func TestNewRSRField1(t *testing.T) { // Normal case - expRSRField1 := &RSRField{Id: "sip_redirected_to", RSRules: []*ReSearchReplace{&ReSearchReplace{regexp.MustCompile(`sip:\+49(\d+)@`), "0$1"}}} + expRSRField1 := &RSRField{Id: "sip_redirected_to", + RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), ReplaceTemplate: "0$1"}}} if rsrField, err := NewRSRField(`~sip_redirected_to:s/sip:\+49(\d+)@/0$1/`); err != nil { t.Error("Unexpected error: ", err.Error()) } else if !reflect.DeepEqual(expRSRField1, rsrField) { @@ -37,7 +38,8 @@ func TestNewRSRField1(t *testing.T) { t.Error("Parse error, field rule does not contain correct number of separators, received: %v", rsrField) } // One extra separator but escaped - expRSRField3 := &RSRField{Id: "sip_redirected_to", RSRules: []*ReSearchReplace{&ReSearchReplace{regexp.MustCompile(`sip:\+49(\d+)\/@`), "0$1"}}} + expRSRField3 := &RSRField{Id: "sip_redirected_to", + RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)\/@`), ReplaceTemplate: "0$1"}}} if rsrField, err := NewRSRField(`~sip_redirected_to:s/sip:\+49(\d+)\/@/0$1/`); err != nil { t.Error("Unexpected error: ", err.Error()) } else if !reflect.DeepEqual(expRSRField3, rsrField) { @@ -46,7 +48,8 @@ func TestNewRSRField1(t *testing.T) { } func TestNewRSRFieldDDz(t *testing.T) { - expectRSRField := &RSRField{Id: "effective_caller_id_number", RSRules: []*ReSearchReplace{&ReSearchReplace{regexp.MustCompile(`(\d+)`), "+$1"}}} + expectRSRField := &RSRField{Id: "effective_caller_id_number", + RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"}}} if rsrField, err := NewRSRField(`~effective_caller_id_number:s/(\d+)/+$1/`); err != nil { t.Error(err) } else if !reflect.DeepEqual(rsrField, expectRSRField) { @@ -55,7 +58,8 @@ func TestNewRSRFieldDDz(t *testing.T) { } func TestNewRSRFieldIvo(t *testing.T) { - expectRSRField := &RSRField{Id: "cost_details", RSRules: []*ReSearchReplace{&ReSearchReplace{regexp.MustCompile(`MatchedDestId":".+_(\s\s\s\s\s)"`), "$1"}}} + expectRSRField := &RSRField{Id: "cost_details", + RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`MatchedDestId":".+_(\s\s\s\s\s)"`), ReplaceTemplate: "$1"}}} if rsrField, err := NewRSRField(`~cost_details:s/MatchedDestId":".+_(\s\s\s\s\s)"/$1/`); err != nil { t.Error(err) } else if !reflect.DeepEqual(rsrField, expectRSRField) { @@ -65,8 +69,8 @@ func TestNewRSRFieldIvo(t *testing.T) { func TestConvertPlusNationalAnd00(t *testing.T) { expectRSRField := &RSRField{Id: "effective_caller_id_number", RSRules: []*ReSearchReplace{ - &ReSearchReplace{regexp.MustCompile(`\+49(\d+)`), "0$1"}, - &ReSearchReplace{regexp.MustCompile(`\+(\d+)`), "00$1"}}} + &ReSearchReplace{SearchRegexp: regexp.MustCompile(`\+49(\d+)`), ReplaceTemplate: "0$1"}, + &ReSearchReplace{SearchRegexp: regexp.MustCompile(`\+(\d+)`), ReplaceTemplate: "00$1"}}} rsrField, err := NewRSRField(`~effective_caller_id_number:s/\+49(\d+)/0$1/:s/\+(\d+)/00$1/`) if err != nil { t.Error(err) @@ -100,7 +104,7 @@ func TestRSRParseStatic(t *testing.T) { func TestConvertDurToSecs(t *testing.T) { expectRSRField := &RSRField{Id: "9", RSRules: []*ReSearchReplace{ - &ReSearchReplace{regexp.MustCompile(`^(\d+)$`), "${1}s"}}} + &ReSearchReplace{SearchRegexp: regexp.MustCompile(`^(\d+)$`), ReplaceTemplate: "${1}s"}}} rsrField, err := NewRSRField(`~9:s/^(\d+)$/${1}s/`) if err != nil { t.Error(err) @@ -114,7 +118,7 @@ func TestConvertDurToSecs(t *testing.T) { func TestPrefix164(t *testing.T) { expectRSRField := &RSRField{Id: "0", RSRules: []*ReSearchReplace{ - &ReSearchReplace{regexp.MustCompile(`^([1-9]\d+)$`), "+$1"}}} + &ReSearchReplace{SearchRegexp: regexp.MustCompile(`^([1-9]\d+)$`), ReplaceTemplate: "+$1"}}} rsrField, err := NewRSRField(`~0:s/^([1-9]\d+)$/+$1/`) if err != nil { t.Error(err) @@ -125,3 +129,14 @@ func TestPrefix164(t *testing.T) { t.Errorf("Expecting: +4986517174960, received: %s", parsedVal) } } + +func TestIsStatic(t *testing.T) { + rsr1 := &RSRField{Id: "0", staticValue: "0"} + if !rsr1.IsStatic() { + t.Error("Failed to detect static value.") + } + rsr2 := &RSRField{Id: "0", RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`^([1-9]\d+)$`), ReplaceTemplate: "+$1"}}} + if rsr2.IsStatic() { + t.Error("Non static detected as static value") + } +} diff --git a/utils/storedcdr.go b/utils/storedcdr.go index 733b84d80..2f5f13412 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -137,8 +137,15 @@ func (storedCdr *StoredCdr) PassesFieldFilter(fieldFilter *RSRField) bool { if fieldFilter == nil { return true } - if storedCdr.FieldAsString(&RSRField{Id: fieldFilter.Id}) == storedCdr.FieldAsString(fieldFilter) && len(storedCdr.FieldAsString(fieldFilter)) != 0 { - // Field value must be non empty in order to declare it filtered, otherwise filter makes no sense + if fieldFilter.IsStatic() && storedCdr.FieldAsString(&RSRField{Id: fieldFilter.Id}) == storedCdr.FieldAsString(fieldFilter) { + return true + } + preparedFilter := &RSRField{Id: fieldFilter.Id, RSRules: make([]*ReSearchReplace, len(fieldFilter.RSRules))} // Reset rules so they do not point towards same structures as original fieldFilter + for idx := range fieldFilter.RSRules { + // Hardcode the template with maximum of 5 groups ordered + preparedFilter.RSRules[idx] = &ReSearchReplace{SearchRegexp: fieldFilter.RSRules[idx].SearchRegexp, ReplaceTemplate: "$1$2$3$4$5"} + } + if storedCdr.FieldAsString(preparedFilter) == storedCdr.FieldAsString(fieldFilter) && preparedFilter.RegexpMatched() { return true } return false diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index 6b5c9022a..d6b2f7a84 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -79,8 +79,10 @@ func TestFieldAsString(t *testing.T) { } func TestPassesFieldFilter(t *testing.T) { - cdr := StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", - Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID, + cdr := StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: VOICE, AccId: "dsafdsaf", + CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", + Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), + AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID, Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } if !cdr.PassesFieldFilter(nil) { @@ -90,6 +92,22 @@ func TestPassesFieldFilter(t *testing.T) { if !cdr.PassesFieldFilter(acntPrefxFltr) { t.Error("Not passing filter") } + acntPrefxFltr, _ = NewRSRField(`~account:s/^(10)\d\d$/10/`) + if !cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Not passing valid filter") + } + acntPrefxFltr, _ = NewRSRField(`~account:s/^\d(10)\d$/10/`) + if cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Passing filter") + } + acntPrefxFltr, _ = NewRSRField(`~account:s/^(10)\d\d$/010/`) + if cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Passing filter") + } + acntPrefxFltr, _ = NewRSRField(`~account:s/^1010$/1010/`) + if cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Passing filter") + } torFltr, _ := NewRSRField(`^tor/*voice`) if !cdr.PassesFieldFilter(torFltr) { t.Error("Not passing filter") From 362d2d4b2203d37527b16630d99bfeea2ad3c980 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 7 Jul 2014 18:16:21 +0200 Subject: [PATCH 07/25] Disabling debug message for derived charging filter in mediator --- mediator/mediator.go | 2 +- sessionmanager/fssessionmanager.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mediator/mediator.go b/mediator/mediator.go index d0fccb4f2..e4db5f1b2 100644 --- a/mediator/mediator.go +++ b/mediator/mediator.go @@ -122,7 +122,7 @@ func (self *Mediator) RateCdr(storedCdr *utils.StoredCdr) error { for _, dc := range dcs { dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) if !storedCdr.PassesFieldFilter(dcRunFilter) { - engine.Logger.Info(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) + //engine.Logger.Info(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) continue } dcReqTypeFld, _ := utils.NewRSRField(dc.ReqTypeField) diff --git a/sessionmanager/fssessionmanager.go b/sessionmanager/fssessionmanager.go index 161fbfeef..f203fdb3c 100644 --- a/sessionmanager/fssessionmanager.go +++ b/sessionmanager/fssessionmanager.go @@ -162,9 +162,10 @@ func (sm *FSSessionManager) OnChannelPark(ev Event) { dcs, _ = dcs.AppendDefaultRun() for _, dc := range dcs { dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) - if dcRunFilter != nil && ev.ParseEventValue(&utils.RSRField{Id: dcRunFilter.Id}) != ev.ParseEventValue(dcRunFilter) { - engine.Logger.Info(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) - } + /*if dcRunFilter != nil && ev.ParseEventValue(&utils.RSRField{Id: dcRunFilter.Id}) != ev.ParseEventValue(dcRunFilter) { //ToDo: Fix here filter + //engine.Logger.Info(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) + continue + }*/ startTime, err := ev.GetAnswerTime(PARK_TIME) if err != nil { engine.Logger.Err("Error parsing answer event start time, using time.Now!") From a13ebf828cedf86a5e35297fcb99349a499a6b96 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 7 Jul 2014 18:17:09 +0200 Subject: [PATCH 08/25] Disabling derived charging filter in session manager until better implementation --- sessionmanager/fssessionmanager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sessionmanager/fssessionmanager.go b/sessionmanager/fssessionmanager.go index f203fdb3c..8771f34b4 100644 --- a/sessionmanager/fssessionmanager.go +++ b/sessionmanager/fssessionmanager.go @@ -161,8 +161,8 @@ func (sm *FSSessionManager) OnChannelPark(ev Event) { } dcs, _ = dcs.AppendDefaultRun() for _, dc := range dcs { - dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) - /*if dcRunFilter != nil && ev.ParseEventValue(&utils.RSRField{Id: dcRunFilter.Id}) != ev.ParseEventValue(dcRunFilter) { //ToDo: Fix here filter + /*dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) + if dcRunFilter != nil && ev.ParseEventValue(&utils.RSRField{Id: dcRunFilter.Id}) != ev.ParseEventValue(dcRunFilter) { //ToDo: Fix here filter //engine.Logger.Info(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) continue }*/ From f77e6f03f3f7710b1a41a45c65983a1597953631 Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 7 Jul 2014 19:22:43 +0200 Subject: [PATCH 09/25] Allow derived charging regexp filters with empty templates --- utils/rsrfield.go | 2 +- utils/storedcdr.go | 2 ++ utils/storedcdr_test.go | 31 ++++++++++++++++++++++++++++++- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/utils/rsrfield.go b/utils/rsrfield.go index 56731428c..baf93ca21 100644 --- a/utils/rsrfield.go +++ b/utils/rsrfield.go @@ -46,7 +46,7 @@ func NewRSRField(fldStr string) (*RSRField, error) { return nil, fmt.Errorf("Invalid Search&Replace field rule. %s", fldStr) } rsrField := &RSRField{Id: spltRules[0][1:]} // Original id in form ~hdr_name - rulesRgxp := regexp.MustCompile(`(?:(.+[^\\])\/(.+[^\\])\/){1,}`) + rulesRgxp := regexp.MustCompile(`(?:(.+[^\\])\/(.+[^\\])*\/){1,}`) for _, ruleStr := range spltRules[1:] { // :s/ already removed through split allMatches := rulesRgxp.FindStringSubmatch(ruleStr) if len(allMatches) != 3 { diff --git a/utils/storedcdr.go b/utils/storedcdr.go index 2f5f13412..b08a815c7 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -145,6 +145,8 @@ func (storedCdr *StoredCdr) PassesFieldFilter(fieldFilter *RSRField) bool { // Hardcode the template with maximum of 5 groups ordered preparedFilter.RSRules[idx] = &ReSearchReplace{SearchRegexp: fieldFilter.RSRules[idx].SearchRegexp, ReplaceTemplate: "$1$2$3$4$5"} } + fmt.Printf("For filterId: %s, preparedFilter match result: %s, filter match result: %s\n", + fieldFilter.Id, storedCdr.FieldAsString(preparedFilter), storedCdr.FieldAsString(fieldFilter)) if storedCdr.FieldAsString(preparedFilter) == storedCdr.FieldAsString(fieldFilter) && preparedFilter.RegexpMatched() { return true } diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index d6b2f7a84..5d0fb9142 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -79,7 +79,7 @@ func TestFieldAsString(t *testing.T) { } func TestPassesFieldFilter(t *testing.T) { - cdr := StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: VOICE, AccId: "dsafdsaf", + cdr := &StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID, @@ -122,6 +122,35 @@ func TestPassesFieldFilter(t *testing.T) { } } +func TestPassesFieldFilterDn1(t *testing.T) { + cdr := &StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "futurem0005", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + } + acntPrefxFltr, _ := NewRSRField(`~account:s/^\w+[s,h,m,p]\d{4}$//`) + if !cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Not passing valid filter") + } + cdr = &StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "futurem00005", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + } + if cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Should not pass filter") + } + cdr = &StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "0402129281", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + } + acntPrefxFltr, _ = NewRSRField(`~account:s/^0\d{9}$//`) + if !cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Not passing valid filter") + } + cdr = &StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "04021292812", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + } + if cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Should not pass filter") + } +} + func TestUsageMultiply(t *testing.T) { cdr := StoredCdr{Usage: time.Duration(10) * time.Second} if cdr.UsageMultiply(1024.0, 0); cdr.Usage != time.Duration(10240)*time.Second { From 03fa289a8d90c271fcc93f9804df846bc211c56a Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 7 Jul 2014 19:26:59 +0200 Subject: [PATCH 10/25] Removing debug log --- utils/storedcdr.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils/storedcdr.go b/utils/storedcdr.go index b08a815c7..2f5f13412 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -145,8 +145,6 @@ func (storedCdr *StoredCdr) PassesFieldFilter(fieldFilter *RSRField) bool { // Hardcode the template with maximum of 5 groups ordered preparedFilter.RSRules[idx] = &ReSearchReplace{SearchRegexp: fieldFilter.RSRules[idx].SearchRegexp, ReplaceTemplate: "$1$2$3$4$5"} } - fmt.Printf("For filterId: %s, preparedFilter match result: %s, filter match result: %s\n", - fieldFilter.Id, storedCdr.FieldAsString(preparedFilter), storedCdr.FieldAsString(fieldFilter)) if storedCdr.FieldAsString(preparedFilter) == storedCdr.FieldAsString(fieldFilter) && preparedFilter.RegexpMatched() { return true } From 7bdd49b64970e427ec74c0a9081b58e6adfcfd6b Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 8 Jul 2014 14:32:20 +0200 Subject: [PATCH 11/25] StoredCdr.PassesFieldFilter with grouping in regexp rules, fsevent with PassesFieldFilter method for derived charging --- cdre/cdrexporter.go | 11 +++++++---- mediator/mediator.go | 2 +- sessionmanager/event.go | 1 + sessionmanager/fsevent.go | 20 ++++++++++++++++++++ sessionmanager/fssessionmanager.go | 8 ++++---- utils/consts.go | 1 + utils/storedcdr.go | 16 +++++++++------- utils/storedcdr_test.go | 30 +++++++++++++++++------------- 8 files changed, 60 insertions(+), 29 deletions(-) diff --git a/cdre/cdrexporter.go b/cdre/cdrexporter.go index faf4e10b7..ef6201a99 100644 --- a/cdre/cdrexporter.go +++ b/cdre/cdrexporter.go @@ -117,12 +117,15 @@ func (cdre *CdrExporter) getCdrCostDetails(cgrId, runId string) (string, error) } func (cdre *CdrExporter) getCombimedCdrFieldVal(processedCdr *utils.StoredCdr, filterRule, fieldRule *utils.RSRField) (string, error) { - filterVal := processedCdr.FieldAsString(filterRule) + fltrPass, ftrPassValue := processedCdr.PassesFieldFilter(filterRule) + if !fltrPass { + return "", nil + } for _, cdr := range cdre.cdrs { if cdr.CgrId != processedCdr.CgrId { continue // We only care about cdrs with same primary cdr behind } - if cdr.FieldAsString(&utils.RSRField{Id: filterRule.Id}) == filterVal { + if cdr.FieldAsString(&utils.RSRField{Id: filterRule.Id}) == ftrPassValue { return cdr.FieldAsString(fieldRule), nil } } @@ -141,7 +144,7 @@ func (cdre *CdrExporter) getDateTimeFieldVal(cdr *utils.StoredCdr, fltrRl, field if fieldRl == nil { return "", nil } - if fltrRl != nil && cdr.FieldAsString(&utils.RSRField{Id: fltrRl.Id}) != cdr.FieldAsString(fltrRl) { + if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass { return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl) } if len(layout) == 0 { @@ -159,7 +162,7 @@ func (cdre *CdrExporter) cdrFieldValue(cdr *utils.StoredCdr, fltrRl, rsrFld *uti if rsrFld == nil { return "", nil } - if fltrRl != nil && cdr.FieldAsString(&utils.RSRField{Id: fltrRl.Id}) != cdr.FieldAsString(fltrRl) { + if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass { return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl) } if len(layout) == 0 { diff --git a/mediator/mediator.go b/mediator/mediator.go index e4db5f1b2..796b7416b 100644 --- a/mediator/mediator.go +++ b/mediator/mediator.go @@ -121,7 +121,7 @@ func (self *Mediator) RateCdr(storedCdr *utils.StoredCdr) error { } for _, dc := range dcs { dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) - if !storedCdr.PassesFieldFilter(dcRunFilter) { + if fltrPass, _ := storedCdr.PassesFieldFilter(dcRunFilter); !fltrPass { //engine.Logger.Info(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) continue } diff --git a/sessionmanager/event.go b/sessionmanager/event.go index 4b4f1c4e3..b7db11c4a 100644 --- a/sessionmanager/event.go +++ b/sessionmanager/event.go @@ -42,4 +42,5 @@ type Event interface { GetDuration(string) (time.Duration, error) MissingParameter() bool ParseEventValue(*utils.RSRField) string + PassesFieldFilter(*utils.RSRField) (bool, string) } diff --git a/sessionmanager/fsevent.go b/sessionmanager/fsevent.go index 3f75d983b..76a0cdde8 100644 --- a/sessionmanager/fsevent.go +++ b/sessionmanager/fsevent.go @@ -249,3 +249,23 @@ func (fsev FSEvent) ParseEventValue(rsrFld *utils.RSRField) string { return rsrFld.ParseValue(fsev[rsrFld.Id]) } } + +func (fsev FSEvent) PassesFieldFilter(fieldFilter *utils.RSRField) (bool, string) { + if fieldFilter == nil { + return true, "" + } + if fieldFilter.IsStatic() && fsev.ParseEventValue(&utils.RSRField{Id: fieldFilter.Id}) == fsev.ParseEventValue(fieldFilter) { + return true, fsev.ParseEventValue(&utils.RSRField{Id: fieldFilter.Id}) + } + preparedFilter := &utils.RSRField{Id: fieldFilter.Id, RSRules: make([]*utils.ReSearchReplace, len(fieldFilter.RSRules))} // Reset rules so they do not point towards same structures as original fieldFilter + for idx := range fieldFilter.RSRules { + // Hardcode the template with maximum of 5 groups ordered + preparedFilter.RSRules[idx] = &utils.ReSearchReplace{SearchRegexp: fieldFilter.RSRules[idx].SearchRegexp, ReplaceTemplate: utils.FILTER_REGEXP_TPL} + } + preparedVal := fsev.ParseEventValue(preparedFilter) + filteredValue := fsev.ParseEventValue(fieldFilter) + if preparedFilter.RegexpMatched() && (len(preparedVal) == 0 || preparedVal == filteredValue) { + return true, filteredValue + } + return false, "" +} diff --git a/sessionmanager/fssessionmanager.go b/sessionmanager/fssessionmanager.go index 8771f34b4..9ce699c8c 100644 --- a/sessionmanager/fssessionmanager.go +++ b/sessionmanager/fssessionmanager.go @@ -161,11 +161,11 @@ func (sm *FSSessionManager) OnChannelPark(ev Event) { } dcs, _ = dcs.AppendDefaultRun() for _, dc := range dcs { - /*dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) - if dcRunFilter != nil && ev.ParseEventValue(&utils.RSRField{Id: dcRunFilter.Id}) != ev.ParseEventValue(dcRunFilter) { //ToDo: Fix here filter - //engine.Logger.Info(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) + dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) + if fltrPass, _ := ev.PassesFieldFilter(dcRunFilter); !fltrPass { + //engine.Logger.Debug(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) continue - }*/ + } startTime, err := ev.GetAnswerTime(PARK_TIME) if err != nil { engine.Logger.Err("Error parsing answer event start time, using time.Now!") diff --git a/utils/consts.go b/utils/consts.go index f3dd508d6..4626d5471 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -122,6 +122,7 @@ const ( CDRFIELD = "cdrfield" ASR = "ASR" ACD = "ACD" + FILTER_REGEXP_TPL = "$1$2$3$4$5" ) var ( diff --git a/utils/storedcdr.go b/utils/storedcdr.go index 2f5f13412..518a2a124 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -133,22 +133,24 @@ func (storedCdr *StoredCdr) FieldAsString(rsrFld *RSRField) string { } } -func (storedCdr *StoredCdr) PassesFieldFilter(fieldFilter *RSRField) bool { +func (storedCdr *StoredCdr) PassesFieldFilter(fieldFilter *RSRField) (bool, string) { if fieldFilter == nil { - return true + return true, "" } if fieldFilter.IsStatic() && storedCdr.FieldAsString(&RSRField{Id: fieldFilter.Id}) == storedCdr.FieldAsString(fieldFilter) { - return true + return true, storedCdr.FieldAsString(&RSRField{Id: fieldFilter.Id}) } preparedFilter := &RSRField{Id: fieldFilter.Id, RSRules: make([]*ReSearchReplace, len(fieldFilter.RSRules))} // Reset rules so they do not point towards same structures as original fieldFilter for idx := range fieldFilter.RSRules { // Hardcode the template with maximum of 5 groups ordered - preparedFilter.RSRules[idx] = &ReSearchReplace{SearchRegexp: fieldFilter.RSRules[idx].SearchRegexp, ReplaceTemplate: "$1$2$3$4$5"} + preparedFilter.RSRules[idx] = &ReSearchReplace{SearchRegexp: fieldFilter.RSRules[idx].SearchRegexp, ReplaceTemplate: FILTER_REGEXP_TPL} } - if storedCdr.FieldAsString(preparedFilter) == storedCdr.FieldAsString(fieldFilter) && preparedFilter.RegexpMatched() { - return true + preparedVal := storedCdr.FieldAsString(preparedFilter) + filteredValue := storedCdr.FieldAsString(fieldFilter) + if preparedFilter.RegexpMatched() && (len(preparedVal) == 0 || preparedVal == filteredValue) { + return true, filteredValue } - return false + return false, "" } func (storedCdr *StoredCdr) AsStoredCdr() *StoredCdr { diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index 5d0fb9142..448b904b7 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -85,39 +85,39 @@ func TestPassesFieldFilter(t *testing.T) { AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID, Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } - if !cdr.PassesFieldFilter(nil) { + if pass, _ := cdr.PassesFieldFilter(nil); !pass { t.Error("Not passing filter") } acntPrefxFltr, _ := NewRSRField(`~account:s/(.+)/1001/`) - if !cdr.PassesFieldFilter(acntPrefxFltr) { + if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass { t.Error("Not passing filter") } acntPrefxFltr, _ = NewRSRField(`~account:s/^(10)\d\d$/10/`) - if !cdr.PassesFieldFilter(acntPrefxFltr) { + if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass { t.Error("Not passing valid filter") } acntPrefxFltr, _ = NewRSRField(`~account:s/^\d(10)\d$/10/`) - if cdr.PassesFieldFilter(acntPrefxFltr) { + if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { t.Error("Passing filter") } acntPrefxFltr, _ = NewRSRField(`~account:s/^(10)\d\d$/010/`) - if cdr.PassesFieldFilter(acntPrefxFltr) { + if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { t.Error("Passing filter") } acntPrefxFltr, _ = NewRSRField(`~account:s/^1010$/1010/`) - if cdr.PassesFieldFilter(acntPrefxFltr) { + if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { t.Error("Passing filter") } torFltr, _ := NewRSRField(`^tor/*voice`) - if !cdr.PassesFieldFilter(torFltr) { + if pass, _ := cdr.PassesFieldFilter(torFltr); !pass { t.Error("Not passing filter") } torFltr, _ = NewRSRField(`^tor/*data`) - if cdr.PassesFieldFilter(torFltr) { + if pass, _ := cdr.PassesFieldFilter(torFltr); pass { t.Error("Passing filter") } inexistentFieldFltr, _ := NewRSRField(`^fakefield/fakevalue`) - if cdr.PassesFieldFilter(inexistentFieldFltr) { + if pass, _ := cdr.PassesFieldFilter(inexistentFieldFltr); pass { t.Error("Passing filter") } } @@ -127,26 +127,30 @@ func TestPassesFieldFilterDn1(t *testing.T) { ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } acntPrefxFltr, _ := NewRSRField(`~account:s/^\w+[s,h,m,p]\d{4}$//`) - if !cdr.PassesFieldFilter(acntPrefxFltr) { + if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass { t.Error("Not passing valid filter") } cdr = &StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "futurem00005", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } - if cdr.PassesFieldFilter(acntPrefxFltr) { + if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { t.Error("Should not pass filter") } cdr = &StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "0402129281", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } acntPrefxFltr, _ = NewRSRField(`~account:s/^0\d{9}$//`) - if !cdr.PassesFieldFilter(acntPrefxFltr) { + if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass { t.Error("Not passing valid filter") } + acntPrefxFltr, _ = NewRSRField(`~account:s/^0(\d{9})$/placeholder/`) + if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { + t.Error("Should not pass filter") + } cdr = &StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "04021292812", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } - if cdr.PassesFieldFilter(acntPrefxFltr) { + if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { t.Error("Should not pass filter") } } From 6d00cce055759986878b5f73c56601cb4fb19554 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 8 Jul 2014 18:55:37 +0200 Subject: [PATCH 12/25] Adding ParseRSRFields for list of RSRFields, modifying NewRSRField static format to add suffix / --- utils/derivedchargers_test.go | 8 ++++---- utils/rsrfield.go | 25 +++++++++++++++++++++++-- utils/rsrfield_test.go | 16 +++++++++++++++- utils/storedcdr_test.go | 6 +++--- 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/utils/derivedchargers_test.go b/utils/derivedchargers_test.go index 52e2f1566..0a079a109 100644 --- a/utils/derivedchargers_test.go +++ b/utils/derivedchargers_test.go @@ -67,7 +67,7 @@ func TestNewDerivedCharger(t *testing.T) { } edc2 := &DerivedCharger{ RunId: "test2", - RunFilter: "^cdr_source/tdm_cdrs", + RunFilter: "^cdr_source/tdm_cdrs/", ReqTypeField: "~reqtype2:s/sip:(.+)/$1/", DirectionField: "~direction2:s/sip:(.+)/$1/", TenantField: "~tenant2:s/sip:(.+)/$1/", @@ -79,7 +79,7 @@ func TestNewDerivedCharger(t *testing.T) { AnswerTimeField: "~answertime2:s/sip:(.+)/$1/", UsageField: "~duration2:s/sip:(.+)/$1/", } - edc2.rsrRunFilter, _ = NewRSRField("^cdr_source/tdm_cdrs") + edc2.rsrRunFilter, _ = NewRSRField("^cdr_source/tdm_cdrs/") edc2.rsrReqTypeField, _ = NewRSRField("~reqtype2:s/sip:(.+)/$1/") edc2.rsrDirectionField, _ = NewRSRField("~direction2:s/sip:(.+)/$1/") edc2.rsrTenantField, _ = NewRSRField("~tenant2:s/sip:(.+)/$1/") @@ -91,7 +91,7 @@ func TestNewDerivedCharger(t *testing.T) { edc2.rsrAnswerTimeField, _ = NewRSRField("~answertime2:s/sip:(.+)/$1/") edc2.rsrUsageField, _ = NewRSRField("~duration2:s/sip:(.+)/$1/") if dc2, err := NewDerivedCharger("test2", - "^cdr_source/tdm_cdrs", + "^cdr_source/tdm_cdrs/", "~reqtype2:s/sip:(.+)/$1/", "~direction2:s/sip:(.+)/$1/", "~tenant2:s/sip:(.+)/$1/", @@ -102,7 +102,7 @@ func TestNewDerivedCharger(t *testing.T) { "~setuptime2:s/sip:(.+)/$1/", "~answertime2:s/sip:(.+)/$1/", "~duration2:s/sip:(.+)/$1/"); err != nil { - t.Error("Unexpected error", err.Error) + t.Error("Unexpected error", err) } else if !reflect.DeepEqual(edc2, dc2) { t.Errorf("Expecting: %v, received: %v", edc2, dc2) } diff --git a/utils/rsrfield.go b/utils/rsrfield.go index baf93ca21..6de14784a 100644 --- a/utils/rsrfield.go +++ b/utils/rsrfield.go @@ -30,8 +30,11 @@ func NewRSRField(fldStr string) (*RSRField, error) { } if strings.HasPrefix(fldStr, STATIC_VALUE_PREFIX) { // Special case when RSR is defined as static header/value var staticHdr, staticVal string - if splt := strings.Split(fldStr, HDR_VAL_SEP); len(splt) == 2 { // Using | as separator since ':' is often use in date/time fields - staticHdr, staticVal = splt[0][1:], splt[1] + if splt := strings.Split(fldStr, "/"); len(splt) == 3 { // Using / as separator since ':' is often use in date/time fields + if len(splt[2]) != 0 { // Last split has created empty element + return nil, fmt.Errorf("Invalid static header/value combination: %s", fldStr) + } + staticHdr, staticVal = splt[0][1:], splt[1] // Strip the / suffix } else { staticHdr, staticVal = splt[0][1:], splt[0][1:] // If no split, header will remain as original, value as header without the prefix } @@ -95,3 +98,21 @@ func (rsrf *RSRField) RegexpMatched() bool { // Investigate whether we had a reg } return false } + +// Parses list of RSRFields, used for example as multiple filters in derived charging +func ParseRSRFields(fldsStr string) ([]*RSRField, error) { + //rsrRlsPattern := regexp.MustCompile(`^(~\w+:s/.+/.*/)|(\^.+(/.+/)?)(;(~\w+:s/.+/.*/)|(\^.+(/.+/)?))*$`) //ToDo:Fix here rule able to confirm the content + rulesSplt := strings.Split(fldsStr, ";") + rsrFields := make([]*RSRField, len(rulesSplt)) + for idx, ruleStr := range rulesSplt { + if !strings.HasSuffix(ruleStr, "/") { + return nil, fmt.Errorf("Invalid RSRField string: %s", ruleStr) + } + if rsrField, err := NewRSRField(ruleStr); err != nil { + return nil, err + } else { + rsrFields[idx] = rsrField + } + } + return rsrFields, nil +} diff --git a/utils/rsrfield_test.go b/utils/rsrfield_test.go index 7b215f161..985318003 100644 --- a/utils/rsrfield_test.go +++ b/utils/rsrfield_test.go @@ -86,7 +86,7 @@ func TestConvertPlusNationalAnd00(t *testing.T) { } func TestRSRParseStatic(t *testing.T) { - if rsrField, err := NewRSRField("^static_header/static_value"); err != nil { + if rsrField, err := NewRSRField("^static_header/static_value/"); err != nil { t.Error(err) } else if !reflect.DeepEqual(rsrField, &RSRField{Id: "static_header", staticValue: "static_value"}) { t.Errorf("Unexpected RSRField received: %v", rsrField) @@ -140,3 +140,17 @@ func TestIsStatic(t *testing.T) { t.Error("Non static detected as static value") } } + +func TestParseRSRFields(t *testing.T) { + fieldsStr1 := `~account:s/^\w+[mpls]\d{6}$//;~subject:s/^0\d{9}$//;^destination/+4912345/;~mediation_runid:s/^default$/default/` + rsrFld1, _ := NewRSRField(`~account:s/^\w+[mpls]\d{6}$//`) + rsrFld2, _ := NewRSRField(`~subject:s/^0\d{9}$//`) + rsrFld3, _ := NewRSRField(`^destination/+4912345/`) + rsrFld4, _ := NewRSRField(`~mediation_runid:s/^default$/default/`) + eRSRFields := []*RSRField{rsrFld1, rsrFld2, rsrFld3, rsrFld4} + if rsrFlds, err := ParseRSRFields(fieldsStr1); err != nil { + t.Error("Unexpected error: ", err) + } else if !reflect.DeepEqual(eRSRFields, rsrFlds) { + t.Errorf("Expecting: %v, received: %v", eRSRFields, rsrFlds) + } +} diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index 448b904b7..e71a96ae2 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -108,15 +108,15 @@ func TestPassesFieldFilter(t *testing.T) { if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { t.Error("Passing filter") } - torFltr, _ := NewRSRField(`^tor/*voice`) + torFltr, _ := NewRSRField(`^tor/*voice/`) if pass, _ := cdr.PassesFieldFilter(torFltr); !pass { t.Error("Not passing filter") } - torFltr, _ = NewRSRField(`^tor/*data`) + torFltr, _ = NewRSRField(`^tor/*data/`) if pass, _ := cdr.PassesFieldFilter(torFltr); pass { t.Error("Passing filter") } - inexistentFieldFltr, _ := NewRSRField(`^fakefield/fakevalue`) + inexistentFieldFltr, _ := NewRSRField(`^fakefield/fakevalue/`) if pass, _ := cdr.PassesFieldFilter(inexistentFieldFltr); pass { t.Error("Passing filter") } From 090f7706e8fc1494dbca7267767f5916a74de17c Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 9 Jul 2014 11:39:10 +0200 Subject: [PATCH 13/25] Small test fix --- cdrs/fscdr_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdrs/fscdr_test.go b/cdrs/fscdr_test.go index 1c4ffe2fb..243a123bf 100644 --- a/cdrs/fscdr_test.go +++ b/cdrs/fscdr_test.go @@ -74,7 +74,7 @@ func TestSearchExtraFieldLast(t *testing.T) { func TestSearchExtraField(t *testing.T) { fsCdr, _ := NewFSCdr(body) rsrSt1, _ := utils.NewRSRField("^injected_value") - rsrSt2, _ := utils.NewRSRField("^injected_hdr/injected_value") + rsrSt2, _ := utils.NewRSRField("^injected_hdr/injected_value/") cfg.CDRSExtraFields = []*utils.RSRField{&utils.RSRField{Id: "caller_id_name"}, rsrSt1, rsrSt2} extraFields := fsCdr.getExtraFields() if len(extraFields) != 3 || extraFields["caller_id_name"] != "dan" || From b5f26c00b891facba9696ec60292ca527086e495 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 9 Jul 2014 12:31:45 +0200 Subject: [PATCH 14/25] Changed cgrid hash to build using SetupTime.UTC() --- cdrs/fscdr.go | 2 +- cdrs/fscdr_test.go | 2 +- sessionmanager/fsevent.go | 3 ++- sessionmanager/fsevent_test.go | 6 +++++- utils/cgrcdr.go | 2 +- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/cdrs/fscdr.go b/cdrs/fscdr.go index 95f90c0e9..f7437661c 100644 --- a/cdrs/fscdr.go +++ b/cdrs/fscdr.go @@ -75,7 +75,7 @@ type FSCdr struct { func (fsCdr FSCdr) getCgrId() string { setupTime, _ := utils.ParseTimeDetectLayout(fsCdr.vars[FS_SETUP_TIME]) - return utils.Sha1(fsCdr.vars[FS_UUID], setupTime.String()) + return utils.Sha1(fsCdr.vars[FS_UUID], setupTime.UTC().String()) } func (fsCdr FSCdr) getExtraFields() map[string]string { diff --git a/cdrs/fscdr_test.go b/cdrs/fscdr_test.go index 243a123bf..b26995c26 100644 --- a/cdrs/fscdr_test.go +++ b/cdrs/fscdr_test.go @@ -54,7 +54,7 @@ func TestCDRFields(t *testing.T) { } setupTime, _ := utils.ParseTimeDetectLayout(fsCdr.vars[FS_SETUP_TIME]) answerTime, _ := utils.ParseTimeDetectLayout(fsCdr.vars[FS_ANSWER_TIME]) - expctStoredCdr := &utils.StoredCdr{CgrId: utils.Sha1("01df56f4-d99a-4ef6-b7fe-b924b2415b7f", setupTime.String()), TOR: utils.VOICE, AccId: "01df56f4-d99a-4ef6-b7fe-b924b2415b7f", + expctStoredCdr := &utils.StoredCdr{CgrId: utils.Sha1("01df56f4-d99a-4ef6-b7fe-b924b2415b7f", setupTime.UTC().String()), TOR: utils.VOICE, AccId: "01df56f4-d99a-4ef6-b7fe-b924b2415b7f", CdrHost: "127.0.0.1", CdrSource: "freeswitch_json", Direction: "*out", Category: "call", ReqType: utils.RATED, Tenant: "ipbx.itsyscom.com", Account: "dan", Subject: "dan", Destination: "+4986517174963", SetupTime: setupTime, AnswerTime: answerTime, Usage: time.Duration(4) * time.Second, ExtraFields: map[string]string{"sip_user_agent": "Jitsi2.2.4603.9615Linux"}, Cost: -1} diff --git a/sessionmanager/fsevent.go b/sessionmanager/fsevent.go index 76a0cdde8..71e413ce0 100644 --- a/sessionmanager/fsevent.go +++ b/sessionmanager/fsevent.go @@ -137,7 +137,7 @@ func (fsev FSEvent) GetCategory(fieldName string) string { } func (fsev FSEvent) GetCgrId() string { setupTime, _ := fsev.GetSetupTime(utils.META_DEFAULT) - return utils.Sha1(fsev[UUID], setupTime.String()) + return utils.Sha1(fsev[UUID], setupTime.UTC().String()) } func (fsev FSEvent) GetUUID() string { return fsev[UUID] @@ -251,6 +251,7 @@ func (fsev FSEvent) ParseEventValue(rsrFld *utils.RSRField) string { } func (fsev FSEvent) PassesFieldFilter(fieldFilter *utils.RSRField) (bool, string) { + // Keep in sync (or merge) with StoredCdr.PassesFieldFielter() if fieldFilter == nil { return true, "" } diff --git a/sessionmanager/fsevent_test.go b/sessionmanager/fsevent_test.go index 04ceab740..da9b23ce2 100644 --- a/sessionmanager/fsevent_test.go +++ b/sessionmanager/fsevent_test.go @@ -504,7 +504,7 @@ func TestParseEventValue(t *testing.T) { cfg, _ = config.NewDefaultCGRConfig() config.SetCgrConfig(cfg) ev := new(FSEvent).New(hangupEv) - if cgrid := ev.ParseEventValue(&utils.RSRField{Id: utils.CGRID}); cgrid != "8b1ca78a9bbaa42c811e60b974188197c425dbe7" { + if cgrid := ev.ParseEventValue(&utils.RSRField{Id: utils.CGRID}); cgrid != "873e5bf7903978f305f7d8fed3f92f968cf82873" { t.Error("Unexpected cgrid parsed", cgrid) } if tor := ev.ParseEventValue(&utils.RSRField{Id: utils.TOR}); tor != utils.VOICE { @@ -559,3 +559,7 @@ func TestParseEventValue(t *testing.T) { t.Error("Unexpected result parsed", parsed) } } + +func TestPassesFieldFilter(t *testing.T) { + +} diff --git a/utils/cgrcdr.go b/utils/cgrcdr.go index 5007f599f..0f424d560 100644 --- a/utils/cgrcdr.go +++ b/utils/cgrcdr.go @@ -40,7 +40,7 @@ type CgrCdr map[string]string func (cgrCdr CgrCdr) getCgrId() string { setupTime, _ := ParseTimeDetectLayout(cgrCdr[SETUP_TIME]) - return Sha1(cgrCdr[ACCID], setupTime.String()) + return Sha1(cgrCdr[ACCID], setupTime.UTC().String()) } func (cgrCdr CgrCdr) getExtraFields() map[string]string { From 3a8d78e798b89c6a8c7fc2d4865a75379df7c937 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 9 Jul 2014 12:57:06 +0200 Subject: [PATCH 15/25] fsevent.ParseEventValue tests refactored to cope with time differences on different build machines --- sessionmanager/fsevent_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sessionmanager/fsevent_test.go b/sessionmanager/fsevent_test.go index da9b23ce2..9d7969066 100644 --- a/sessionmanager/fsevent_test.go +++ b/sessionmanager/fsevent_test.go @@ -540,11 +540,13 @@ func TestParseEventValue(t *testing.T) { if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.DESTINATION}); parsed != "1002" { t.Error("Unexpected result parsed", parsed) } - if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.SETUP_TIME}); parsed != "2014-04-25 18:08:27 +0200 CEST" { - t.Error("Unexpected result parsed", parsed) + sTime, _ := utils.ParseTimeDetectLayout("1398442107770704"[:len("1398442107770704")-6]) // We discard nanoseconds information so we can correlate csv + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.SETUP_TIME}); parsed != sTime.String() { + t.Errorf("Expecting: %s, parsed: %s", sTime.String(), parsed) } - if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.ANSWER_TIME}); parsed != "2014-04-25 18:08:40 +0200 CEST" { - t.Error("Unexpected result parsed", parsed) + aTime, _ := utils.ParseTimeDetectLayout("1398442120831856"[:len("1398442120831856")-6]) + if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.ANSWER_TIME}); parsed != aTime.String() { + t.Errorf("Expecting: %s, parsed: %s", aTime.String(), parsed) } if parsed := ev.ParseEventValue(&utils.RSRField{Id: utils.USAGE}); parsed != "5000000000" { t.Error("Unexpected result parsed", parsed) From 0986c1371bdab495d7400515815574cd7f0251a4 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 9 Jul 2014 13:39:57 +0200 Subject: [PATCH 16/25] Adding ev.PassesFieldFilter tests --- sessionmanager/fsevent_test.go | 49 ++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/sessionmanager/fsevent_test.go b/sessionmanager/fsevent_test.go index 9d7969066..29dc1a9dd 100644 --- a/sessionmanager/fsevent_test.go +++ b/sessionmanager/fsevent_test.go @@ -562,6 +562,51 @@ func TestParseEventValue(t *testing.T) { } } -func TestPassesFieldFilter(t *testing.T) { - +func TestPassesFieldFilterDn1(t *testing.T) { + body := `Event-Name: RE_SCHEDULE +Core-UUID: 792e181c-b6e6-499c-82a1-52a778e7d82d +FreeSWITCH-Hostname: h1.ip-switch.net +FreeSWITCH-Switchname: h1.ip-switch.net +FreeSWITCH-IPv4: 88.198.12.156 +Caller-Username: futurem0005` + ev := new(FSEvent).New(body) + acntPrefxFltr, _ := utils.NewRSRField(`~account:s/^\w+[s,h,m,p]\d{4}$//`) + if pass, _ := ev.PassesFieldFilter(acntPrefxFltr); !pass { + t.Error("Not passing valid filter") + } + body = `Event-Name: RE_SCHEDULE +Core-UUID: 792e181c-b6e6-499c-82a1-52a778e7d82d +FreeSWITCH-Hostname: h1.ip-switch.net +FreeSWITCH-Switchname: h1.ip-switch.net +FreeSWITCH-IPv4: 88.198.12.156 +Caller-Username: futurem00005` + ev = new(FSEvent).New(body) + if pass, _ := ev.PassesFieldFilter(acntPrefxFltr); pass { + t.Error("Should not pass filter") + } + body = `Event-Name: RE_SCHEDULE +Core-UUID: 792e181c-b6e6-499c-82a1-52a778e7d82d +FreeSWITCH-Hostname: h1.ip-switch.net +FreeSWITCH-Switchname: h1.ip-switch.net +FreeSWITCH-IPv4: 88.198.12.156 +Caller-Username: 0402129281` + ev = new(FSEvent).New(body) + acntPrefxFltr, _ = utils.NewRSRField(`~account:s/^0\d{9}$//`) + if pass, _ := ev.PassesFieldFilter(acntPrefxFltr); !pass { + t.Error("Not passing valid filter") + } + acntPrefxFltr, _ = utils.NewRSRField(`~account:s/^0(\d{9})$/placeholder/`) + if pass, _ := ev.PassesFieldFilter(acntPrefxFltr); pass { + t.Error("Should not pass filter") + } + body = `Event-Name: RE_SCHEDULE +Core-UUID: 792e181c-b6e6-499c-82a1-52a778e7d82d +FreeSWITCH-Hostname: h1.ip-switch.net +FreeSWITCH-Switchname: h1.ip-switch.net +FreeSWITCH-IPv4: 88.198.12.156 +Caller-Username: 04021292812` + ev = new(FSEvent).New(body) + if pass, _ := ev.PassesFieldFilter(acntPrefxFltr); pass { + t.Error("Should not pass filter") + } } From 22670e12a81dafdb42e4b3c7ec0ee286990f05bd Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 9 Jul 2014 15:21:42 +0200 Subject: [PATCH 17/25] Logic to handle multiple derived charging run filters inside mediator and session manager --- apier/derivedcharging.go | 5 +++ config/config_test.go | 2 +- config/helpers.go | 2 +- engine/loader_csv.go | 7 ++-- engine/loader_csv_test.go | 6 ++-- mediator/mediator.go | 9 ++--- sessionmanager/fsevent_test.go | 2 +- sessionmanager/fssessionmanager.go | 11 +++--- utils/consts.go | 2 ++ utils/derivedchargers.go | 56 ++++++++++++++---------------- utils/derivedchargers_test.go | 10 +++--- utils/rsrfield.go | 7 ++-- utils/rsrfield_test.go | 2 +- utils/storedcdr_test.go | 2 +- 14 files changed, 68 insertions(+), 55 deletions(-) diff --git a/apier/derivedcharging.go b/apier/derivedcharging.go index 0c4e5d648..fb3766e1d 100644 --- a/apier/derivedcharging.go +++ b/apier/derivedcharging.go @@ -46,6 +46,11 @@ func (self *ApierV1) SetDerivedChargers(attrs AttrSetDerivedChargers, reply *str if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Category", "Direction", "Account", "Subject", "DerivedChargers"}); len(missing) != 0 { return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing) } + for _, dc := range attrs.DerivedChargers { + if _, err = utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP); err != nil { // Make sure rules are OK before loading in db + return fmt.Errorf("%s:%s", utils.ERR_PARSER_ERROR, err.Error()) + } + } dcKey := utils.DerivedChargersKey(attrs.Direction, attrs.Tenant, attrs.Category, attrs.Account, attrs.Subject) if err := self.AccountDb.SetDerivedChargers(dcKey, attrs.DerivedChargers); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) diff --git a/config/config_test.go b/config/config_test.go index 1c4eaab56..3bf32117b 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -250,7 +250,7 @@ func TestConfigFromFile(t *testing.T) { eCfg.FreeswitchServer = "test" eCfg.FreeswitchPass = "test" eCfg.FreeswitchReconnects = 99 - eCfg.DerivedChargers = utils.DerivedChargers{&utils.DerivedCharger{RunId: "test", RunFilter: "", ReqTypeField: "test", DirectionField: "test", TenantField: "test", + eCfg.DerivedChargers = utils.DerivedChargers{&utils.DerivedCharger{RunId: "test", RunFilters: "", ReqTypeField: "test", DirectionField: "test", TenantField: "test", CategoryField: "test", AccountField: "test", SubjectField: "test", DestinationField: "test", SetupTimeField: "test", AnswerTimeField: "test", UsageField: "test"}} eCfg.CombinedDerivedChargers = true eCfg.HistoryAgentEnabled = true diff --git a/config/helpers.go b/config/helpers.go index 02fad02ce..4b9bfde4b 100644 --- a/config/helpers.go +++ b/config/helpers.go @@ -39,7 +39,7 @@ func ConfigSlice(cfgVal string) ([]string, error) { return cfgValStrs, nil } -func ParseRSRFields(configVal string) ([]*utils.RSRField, error) { +func ParseRSRFields(configVal string) ([]*utils.RSRField, error) { //ToDo: Unify it with the Parser inside RSRField cfgValStrs := strings.Split(configVal, string(utils.CSV_SEP)) if len(cfgValStrs) == 1 && cfgValStrs[0] == "" { // Prevents returning iterable with empty value return []*utils.RSRField{}, nil diff --git a/engine/loader_csv.go b/engine/loader_csv.go index 47f2ab28f..2e657448e 100644 --- a/engine/loader_csv.go +++ b/engine/loader_csv.go @@ -839,10 +839,13 @@ func (csvr *CSVReader) LoadDerivedChargers() (err error) { for record, err := csvReader.Read(); err == nil; record, err = csvReader.Read() { tag := utils.DerivedChargersKey(record[0], record[1], record[2], record[3], record[4]) _, found := csvr.derivedChargers[tag] + if _, err = utils.ParseRSRFields(record[6], utils.INFIELD_SEP); err != nil { // Make sure rules are OK before loading in db + return err + } if found { if csvr.derivedChargers[tag], err = csvr.derivedChargers[tag].Append(&utils.DerivedCharger{ RunId: ValueOrDefault(record[5], "*default"), - RunFilter: record[6], + RunFilters: record[6], ReqTypeField: ValueOrDefault(record[7], "*default"), DirectionField: ValueOrDefault(record[8], "*default"), TenantField: ValueOrDefault(record[9], "*default"), @@ -862,7 +865,7 @@ func (csvr *CSVReader) LoadDerivedChargers() (err error) { } csvr.derivedChargers[tag] = utils.DerivedChargers{&utils.DerivedCharger{ RunId: ValueOrDefault(record[5], "*default"), - RunFilter: record[6], + RunFilters: record[6], ReqTypeField: ValueOrDefault(record[7], "*default"), DirectionField: ValueOrDefault(record[8], "*default"), TenantField: ValueOrDefault(record[9], "*default"), diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 82e62011e..6393a4800 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -178,7 +178,7 @@ vdf,emptyY,*out,TOPUP_EMPTY_AT, derivedCharges = ` #Direction,Tenant,Category,Account,Subject,RunId,RunFilter,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,UsageField -*out,cgrates.org,call,dan,dan,extra1,^filteredHeader1/filterValue1,^prepaid,,,,rif,rif,,,, +*out,cgrates.org,call,dan,dan,extra1,^filteredHeader1/filterValue1/,^prepaid,,,,rif,rif,,,, *out,cgrates.org,call,dan,dan,extra2,,,,,,ivo,ivo,,,, *out,cgrates.org,call,dan,*any,extra1,,,,,,rif2,rif2,,,, ` @@ -932,8 +932,8 @@ func TestLoadDerivedChargers(t *testing.T) { t.Error("Failed to load derivedChargers: ", csvr.derivedChargers) } expCharger1 := utils.DerivedChargers{ - &utils.DerivedCharger{RunId: "extra1", RunFilter: "^filteredHeader1/filterValue1", ReqTypeField: "^prepaid", DirectionField: "*default", TenantField: "*default", - CategoryField: "*default", AccountField: "rif", SubjectField: "rif", DestinationField: "*default", + &utils.DerivedCharger{RunId: "extra1", RunFilters: "^filteredHeader1/filterValue1/", ReqTypeField: "^prepaid", DirectionField: "*default", + TenantField: "*default", CategoryField: "*default", AccountField: "rif", SubjectField: "rif", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"}, &utils.DerivedCharger{RunId: "extra2", ReqTypeField: "*default", DirectionField: "*default", TenantField: "*default", CategoryField: "*default", AccountField: "ivo", SubjectField: "ivo", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"}, diff --git a/mediator/mediator.go b/mediator/mediator.go index 796b7416b..d2005dfcb 100644 --- a/mediator/mediator.go +++ b/mediator/mediator.go @@ -120,10 +120,11 @@ func (self *Mediator) RateCdr(storedCdr *utils.StoredCdr) error { return errors.New(errText) } for _, dc := range dcs { - dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) - if fltrPass, _ := storedCdr.PassesFieldFilter(dcRunFilter); !fltrPass { - //engine.Logger.Info(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) - continue + runFilters, _ := utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP) + for _, dcRunFilter := range runFilters { + if fltrPass, _ := storedCdr.PassesFieldFilter(dcRunFilter); !fltrPass { + continue + } } dcReqTypeFld, _ := utils.NewRSRField(dc.ReqTypeField) dcDirFld, _ := utils.NewRSRField(dc.DirectionField) diff --git a/sessionmanager/fsevent_test.go b/sessionmanager/fsevent_test.go index 29dc1a9dd..cec0dcc66 100644 --- a/sessionmanager/fsevent_test.go +++ b/sessionmanager/fsevent_test.go @@ -570,7 +570,7 @@ FreeSWITCH-Switchname: h1.ip-switch.net FreeSWITCH-IPv4: 88.198.12.156 Caller-Username: futurem0005` ev := new(FSEvent).New(body) - acntPrefxFltr, _ := utils.NewRSRField(`~account:s/^\w+[s,h,m,p]\d{4}$//`) + acntPrefxFltr, _ := utils.NewRSRField(`~account:s/^\w+[shmp]\d{4}$//`) if pass, _ := ev.PassesFieldFilter(acntPrefxFltr); !pass { t.Error("Not passing valid filter") } diff --git a/sessionmanager/fssessionmanager.go b/sessionmanager/fssessionmanager.go index 9ce699c8c..341791f2f 100644 --- a/sessionmanager/fssessionmanager.go +++ b/sessionmanager/fssessionmanager.go @@ -112,7 +112,7 @@ func (sm *FSSessionManager) DisconnectSession(uuid string, notify string) { return } -// Remove session from sessin list, removes all related in case of multiple runs +// Remove session from session list, removes all related in case of multiple runs func (sm *FSSessionManager) RemoveSession(uuid string) { for i, ss := range sm.sessions { if ss.uuid == uuid { @@ -161,10 +161,11 @@ func (sm *FSSessionManager) OnChannelPark(ev Event) { } dcs, _ = dcs.AppendDefaultRun() for _, dc := range dcs { - dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) - if fltrPass, _ := ev.PassesFieldFilter(dcRunFilter); !fltrPass { - //engine.Logger.Debug(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) - continue + runFilters, _ := utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP) + for _, dcRunFilter := range runFilters { + if fltrPass, _ := ev.PassesFieldFilter(dcRunFilter); !fltrPass { + continue + } } startTime, err := ev.GetAnswerTime(PARK_TIME) if err != nil { diff --git a/utils/consts.go b/utils/consts.go index 4626d5471..1b220470c 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -20,6 +20,7 @@ const ( ERR_MANDATORY_IE_MISSING = "MANDATORY_IE_MISSING" ERR_EXISTS = "EXISTS" ERR_BROKEN_REFERENCE = "BROKEN_REFERENCE" + ERR_PARSER_ERROR = "PARSER_ERROR" TBL_TP_TIMINGS = "tp_timings" TBL_TP_DESTINATIONS = "tp_destinations" TBL_TP_RATES = "tp_rates" @@ -69,6 +70,7 @@ const ( COMMENT_CHAR = '#' CSV_SEP = ',' FALLBACK_SEP = ';' + INFIELD_SEP = ";" REGEXP_PREFIX = "~" JSON = "json" MSGPACK = "msgpack" diff --git a/utils/derivedchargers.go b/utils/derivedchargers.go index 296d05d15..38b8d736b 100644 --- a/utils/derivedchargers.go +++ b/utils/derivedchargers.go @@ -24,75 +24,73 @@ import ( ) // Wraps regexp compiling in case of rsr fields -func NewDerivedCharger(runId, runFilter, reqTypeFld, dirFld, tenantFld, catFld, acntFld, subjFld, dstFld, sTimeFld, aTimeFld, durFld string) (dc *DerivedCharger, err error) { +func NewDerivedCharger(runId, runFilters, reqTypeFld, dirFld, tenantFld, catFld, acntFld, subjFld, dstFld, sTimeFld, aTimeFld, durFld string) (dc *DerivedCharger, err error) { if len(runId) == 0 { return nil, errors.New("Empty run id field") } dc = &DerivedCharger{RunId: runId} - dc.RunFilter = runFilter - if strings.HasPrefix(dc.RunFilter, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { - if dc.rsrRunFilter, err = NewRSRField(dc.RunFilter); err != nil { + dc.RunFilters = runFilters + if strings.HasPrefix(dc.RunFilters, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilters, STATIC_VALUE_PREFIX) { + if dc.rsrRunFilters, err = ParseRSRFields(dc.RunFilters, INFIELD_SEP); err != nil { return nil, err - } else if len(dc.rsrRunFilter.Id) == 0 { - return nil, errors.New("Empty filter header.") } } dc.ReqTypeField = reqTypeFld - if strings.HasPrefix(dc.ReqTypeField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { + if strings.HasPrefix(dc.ReqTypeField, REGEXP_PREFIX) || strings.HasPrefix(dc.ReqTypeField, STATIC_VALUE_PREFIX) { if dc.rsrReqTypeField, err = NewRSRField(dc.ReqTypeField); err != nil { return nil, err } } dc.DirectionField = dirFld - if strings.HasPrefix(dc.DirectionField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { + if strings.HasPrefix(dc.DirectionField, REGEXP_PREFIX) || strings.HasPrefix(dc.DirectionField, STATIC_VALUE_PREFIX) { if dc.rsrDirectionField, err = NewRSRField(dc.DirectionField); err != nil { return nil, err } } dc.TenantField = tenantFld - if strings.HasPrefix(dc.TenantField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { + if strings.HasPrefix(dc.TenantField, REGEXP_PREFIX) || strings.HasPrefix(dc.TenantField, STATIC_VALUE_PREFIX) { if dc.rsrTenantField, err = NewRSRField(dc.TenantField); err != nil { return nil, err } } dc.CategoryField = catFld - if strings.HasPrefix(dc.CategoryField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { + if strings.HasPrefix(dc.CategoryField, REGEXP_PREFIX) || strings.HasPrefix(dc.CategoryField, STATIC_VALUE_PREFIX) { if dc.rsrCategoryField, err = NewRSRField(dc.CategoryField); err != nil { return nil, err } } dc.AccountField = acntFld - if strings.HasPrefix(dc.AccountField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { + if strings.HasPrefix(dc.AccountField, REGEXP_PREFIX) || strings.HasPrefix(dc.AccountField, STATIC_VALUE_PREFIX) { if dc.rsrAccountField, err = NewRSRField(dc.AccountField); err != nil { return nil, err } } dc.SubjectField = subjFld - if strings.HasPrefix(dc.SubjectField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { + if strings.HasPrefix(dc.SubjectField, REGEXP_PREFIX) || strings.HasPrefix(dc.SubjectField, STATIC_VALUE_PREFIX) { if dc.rsrSubjectField, err = NewRSRField(dc.SubjectField); err != nil { return nil, err } } dc.DestinationField = dstFld - if strings.HasPrefix(dc.DestinationField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { + if strings.HasPrefix(dc.DestinationField, REGEXP_PREFIX) || strings.HasPrefix(dc.DestinationField, STATIC_VALUE_PREFIX) { if dc.rsrDestinationField, err = NewRSRField(dc.DestinationField); err != nil { return nil, err } } dc.SetupTimeField = sTimeFld - if strings.HasPrefix(dc.SetupTimeField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { + if strings.HasPrefix(dc.SetupTimeField, REGEXP_PREFIX) || strings.HasPrefix(dc.SetupTimeField, STATIC_VALUE_PREFIX) { if dc.rsrSetupTimeField, err = NewRSRField(dc.SetupTimeField); err != nil { return nil, err } } dc.AnswerTimeField = aTimeFld - if strings.HasPrefix(dc.AnswerTimeField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { + if strings.HasPrefix(dc.AnswerTimeField, REGEXP_PREFIX) || strings.HasPrefix(dc.AnswerTimeField, STATIC_VALUE_PREFIX) { if dc.rsrAnswerTimeField, err = NewRSRField(dc.AnswerTimeField); err != nil { return nil, err } } dc.UsageField = durFld - if strings.HasPrefix(dc.UsageField, REGEXP_PREFIX) || strings.HasPrefix(dc.RunFilter, STATIC_VALUE_PREFIX) { + if strings.HasPrefix(dc.UsageField, REGEXP_PREFIX) || strings.HasPrefix(dc.UsageField, STATIC_VALUE_PREFIX) { if dc.rsrUsageField, err = NewRSRField(dc.UsageField); err != nil { return nil, err } @@ -101,19 +99,19 @@ func NewDerivedCharger(runId, runFilter, reqTypeFld, dirFld, tenantFld, catFld, } type DerivedCharger struct { - RunId string // Unique runId in the chain - RunFilter string // Only run the charger if the filter matches - ReqTypeField string // Field containing request type info, number in case of csv source, '^' as prefix in case of static values - DirectionField string // Field containing direction info - TenantField string // Field containing tenant info - CategoryField string // Field containing tor info - AccountField string // Field containing account information - SubjectField string // Field containing subject information - DestinationField string // Field containing destination information - SetupTimeField string // Field containing setup time information - AnswerTimeField string // Field containing answer time information - UsageField string // Field containing usage information - rsrRunFilter *RSRField // Storage for compiled Regexp in case of RSRFields + RunId string // Unique runId in the chain + RunFilters string // Only run the charger if all the filters match + ReqTypeField string // Field containing request type info, number in case of csv source, '^' as prefix in case of static values + DirectionField string // Field containing direction info + TenantField string // Field containing tenant info + CategoryField string // Field containing tor info + AccountField string // Field containing account information + SubjectField string // Field containing subject information + DestinationField string // Field containing destination information + SetupTimeField string // Field containing setup time information + AnswerTimeField string // Field containing answer time information + UsageField string // Field containing usage information + rsrRunFilters []*RSRField // Storage for compiled Regexp in case of RSRFields rsrReqTypeField *RSRField rsrDirectionField *RSRField rsrTenantField *RSRField diff --git a/utils/derivedchargers_test.go b/utils/derivedchargers_test.go index 0a079a109..e8a76647f 100644 --- a/utils/derivedchargers_test.go +++ b/utils/derivedchargers_test.go @@ -47,7 +47,7 @@ func TestAppendDerivedChargers(t *testing.T) { func TestNewDerivedCharger(t *testing.T) { edc1 := &DerivedCharger{ RunId: "test1", - RunFilter: "", + RunFilters: "", ReqTypeField: "reqtype1", DirectionField: "direction1", TenantField: "tenant1", @@ -67,7 +67,7 @@ func TestNewDerivedCharger(t *testing.T) { } edc2 := &DerivedCharger{ RunId: "test2", - RunFilter: "^cdr_source/tdm_cdrs/", + RunFilters: "^cdr_source/tdm_cdrs/", ReqTypeField: "~reqtype2:s/sip:(.+)/$1/", DirectionField: "~direction2:s/sip:(.+)/$1/", TenantField: "~tenant2:s/sip:(.+)/$1/", @@ -79,7 +79,7 @@ func TestNewDerivedCharger(t *testing.T) { AnswerTimeField: "~answertime2:s/sip:(.+)/$1/", UsageField: "~duration2:s/sip:(.+)/$1/", } - edc2.rsrRunFilter, _ = NewRSRField("^cdr_source/tdm_cdrs/") + edc2.rsrRunFilters, _ = ParseRSRFields("^cdr_source/tdm_cdrs/", INFIELD_SEP) edc2.rsrReqTypeField, _ = NewRSRField("~reqtype2:s/sip:(.+)/$1/") edc2.rsrDirectionField, _ = NewRSRField("~direction2:s/sip:(.+)/$1/") edc2.rsrTenantField, _ = NewRSRField("~tenant2:s/sip:(.+)/$1/") @@ -116,7 +116,7 @@ func TestDerivedChargersKey(t *testing.T) { func TestAppendDefaultRun(t *testing.T) { var dc1 DerivedChargers - dcDf := &DerivedCharger{RunId: DEFAULT_RUNID, RunFilter: "", ReqTypeField: META_DEFAULT, DirectionField: META_DEFAULT, + dcDf := &DerivedCharger{RunId: DEFAULT_RUNID, RunFilters: "", ReqTypeField: META_DEFAULT, DirectionField: META_DEFAULT, TenantField: META_DEFAULT, CategoryField: META_DEFAULT, AccountField: META_DEFAULT, SubjectField: META_DEFAULT, DestinationField: META_DEFAULT, SetupTimeField: META_DEFAULT, AnswerTimeField: META_DEFAULT, UsageField: META_DEFAULT} eDc1 := DerivedChargers{dcDf} @@ -124,7 +124,7 @@ func TestAppendDefaultRun(t *testing.T) { t.Error("Unexpected result.") } dc2 := DerivedChargers{ - &DerivedCharger{RunId: "extra1", RunFilter: "", ReqTypeField: "reqtype2", DirectionField: "*default", TenantField: "*default", CategoryField: "*default", + &DerivedCharger{RunId: "extra1", RunFilters: "", ReqTypeField: "reqtype2", DirectionField: "*default", TenantField: "*default", CategoryField: "*default", AccountField: "rif", SubjectField: "rif", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"}, &DerivedCharger{RunId: "extra2", ReqTypeField: "*default", DirectionField: "*default", TenantField: "*default", CategoryField: "*default", AccountField: "ivo", SubjectField: "ivo", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"}, diff --git a/utils/rsrfield.go b/utils/rsrfield.go index 6de14784a..ed1c2ec18 100644 --- a/utils/rsrfield.go +++ b/utils/rsrfield.go @@ -100,9 +100,12 @@ func (rsrf *RSRField) RegexpMatched() bool { // Investigate whether we had a reg } // Parses list of RSRFields, used for example as multiple filters in derived charging -func ParseRSRFields(fldsStr string) ([]*RSRField, error) { +func ParseRSRFields(fldsStr, sep string) ([]*RSRField, error) { //rsrRlsPattern := regexp.MustCompile(`^(~\w+:s/.+/.*/)|(\^.+(/.+/)?)(;(~\w+:s/.+/.*/)|(\^.+(/.+/)?))*$`) //ToDo:Fix here rule able to confirm the content - rulesSplt := strings.Split(fldsStr, ";") + if len(fldsStr) == 0 { + return nil, nil + } + rulesSplt := strings.Split(fldsStr, sep) rsrFields := make([]*RSRField, len(rulesSplt)) for idx, ruleStr := range rulesSplt { if !strings.HasSuffix(ruleStr, "/") { diff --git a/utils/rsrfield_test.go b/utils/rsrfield_test.go index 985318003..1c8c0c271 100644 --- a/utils/rsrfield_test.go +++ b/utils/rsrfield_test.go @@ -148,7 +148,7 @@ func TestParseRSRFields(t *testing.T) { rsrFld3, _ := NewRSRField(`^destination/+4912345/`) rsrFld4, _ := NewRSRField(`~mediation_runid:s/^default$/default/`) eRSRFields := []*RSRField{rsrFld1, rsrFld2, rsrFld3, rsrFld4} - if rsrFlds, err := ParseRSRFields(fieldsStr1); err != nil { + if rsrFlds, err := ParseRSRFields(fieldsStr1, INFIELD_SEP); err != nil { t.Error("Unexpected error: ", err) } else if !reflect.DeepEqual(eRSRFields, rsrFlds) { t.Errorf("Expecting: %v, received: %v", eRSRFields, rsrFlds) diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index e71a96ae2..bfc02bfd9 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -126,7 +126,7 @@ func TestPassesFieldFilterDn1(t *testing.T) { cdr := &StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "futurem0005", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } - acntPrefxFltr, _ := NewRSRField(`~account:s/^\w+[s,h,m,p]\d{4}$//`) + acntPrefxFltr, _ := NewRSRField(`~account:s/^\w+[shmp]\d{4}$//`) if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass { t.Error("Not passing valid filter") } From e808168d52cc49cd23b52faf51795c3c0e8d67e1 Mon Sep 17 00:00:00 2001 From: DanB Date: Wed, 9 Jul 2014 17:49:25 +0200 Subject: [PATCH 18/25] Merging old ParseRSRFields from config with new one in utils --- config/config.go | 2 +- config/helpers.go | 16 ---------------- config/helpers_test.go | 14 -------------- utils/consts.go | 1 + utils/rsrfield.go | 5 ++--- utils/rsrfield_test.go | 11 +++++++++++ 6 files changed, 15 insertions(+), 34 deletions(-) diff --git a/config/config.go b/config/config.go index 24739376e..7e7afd546 100644 --- a/config/config.go +++ b/config/config.go @@ -386,7 +386,7 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { } if hasOpt = c.HasOption("cdrs", "extra_fields"); hasOpt { extraFieldsStr, _ := c.GetString("cdrs", "extra_fields") - if extraFields, err := ParseRSRFields(extraFieldsStr); err != nil { + if extraFields, err := utils.ParseRSRFields(extraFieldsStr, utils.FIELDS_SEP); err != nil { return nil, err } else { cfg.CDRSExtraFields = extraFields diff --git a/config/helpers.go b/config/helpers.go index 4b9bfde4b..b4c061f97 100644 --- a/config/helpers.go +++ b/config/helpers.go @@ -39,22 +39,6 @@ func ConfigSlice(cfgVal string) ([]string, error) { return cfgValStrs, nil } -func ParseRSRFields(configVal string) ([]*utils.RSRField, error) { //ToDo: Unify it with the Parser inside RSRField - cfgValStrs := strings.Split(configVal, string(utils.CSV_SEP)) - if len(cfgValStrs) == 1 && cfgValStrs[0] == "" { // Prevents returning iterable with empty value - return []*utils.RSRField{}, nil - } - rsrFields := make([]*utils.RSRField, len(cfgValStrs)) - for idx, cfgValStr := range cfgValStrs { - if rsrField, err := utils.NewRSRField(cfgValStr); err != nil { - return nil, err - } else { - rsrFields[idx] = rsrField - } - } - return rsrFields, nil -} - // Parse the configuration file and returns utils.DerivedChargers instance if no errors func ParseCfgDerivedCharging(c *conf.ConfigFile) (dcs utils.DerivedChargers, err error) { var runIds, runFilters, reqTypeFlds, directionFlds, tenantFlds, torFlds, acntFlds, subjFlds, dstFlds, sTimeFlds, aTimeFlds, durFlds []string diff --git a/config/helpers_test.go b/config/helpers_test.go index 31aa4747c..a64d02f52 100644 --- a/config/helpers_test.go +++ b/config/helpers_test.go @@ -20,7 +20,6 @@ package config import ( "reflect" - "regexp" "testing" "github.com/cgrates/cgrates/utils" @@ -35,19 +34,6 @@ func TestConfigSlice(t *testing.T) { } } -func TestParseRSRFields(t *testing.T) { - fields := `host,~sip_redirected_to:s/sip:\+49(\d+)@/0$1/,destination` - expectParsedFields := []*utils.RSRField{&utils.RSRField{Id: "host"}, - &utils.RSRField{Id: "sip_redirected_to", - RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), ReplaceTemplate: "0$1"}}}, - &utils.RSRField{Id: "destination"}} - if parsedFields, err := ParseRSRFields(fields); err != nil { - t.Error("Unexpected error: ", err.Error()) - } else if !reflect.DeepEqual(parsedFields, expectParsedFields) { - t.Errorf("Unexpected value of parsed fields") - } -} - func TestParseCfgDerivedCharging(t *testing.T) { eFieldsCfg := []byte(`[derived_charging] run_ids = run1, run2 diff --git a/utils/consts.go b/utils/consts.go index 1b220470c..014fba60a 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -71,6 +71,7 @@ const ( CSV_SEP = ',' FALLBACK_SEP = ';' INFIELD_SEP = ";" + FIELDS_SEP = "," REGEXP_PREFIX = "~" JSON = "json" MSGPACK = "msgpack" diff --git a/utils/rsrfield.go b/utils/rsrfield.go index ed1c2ec18..20ceaaac5 100644 --- a/utils/rsrfield.go +++ b/utils/rsrfield.go @@ -35,6 +35,8 @@ func NewRSRField(fldStr string) (*RSRField, error) { return nil, fmt.Errorf("Invalid static header/value combination: %s", fldStr) } staticHdr, staticVal = splt[0][1:], splt[1] // Strip the / suffix + } else if len(splt) == 2 { + return nil, fmt.Errorf("Invalid RSRField string: %s", fldStr) } else { staticHdr, staticVal = splt[0][1:], splt[0][1:] // If no split, header will remain as original, value as header without the prefix } @@ -108,9 +110,6 @@ func ParseRSRFields(fldsStr, sep string) ([]*RSRField, error) { rulesSplt := strings.Split(fldsStr, sep) rsrFields := make([]*RSRField, len(rulesSplt)) for idx, ruleStr := range rulesSplt { - if !strings.HasSuffix(ruleStr, "/") { - return nil, fmt.Errorf("Invalid RSRField string: %s", ruleStr) - } if rsrField, err := NewRSRField(ruleStr); err != nil { return nil, err } else { diff --git a/utils/rsrfield_test.go b/utils/rsrfield_test.go index 1c8c0c271..9296da4dc 100644 --- a/utils/rsrfield_test.go +++ b/utils/rsrfield_test.go @@ -153,4 +153,15 @@ func TestParseRSRFields(t *testing.T) { } else if !reflect.DeepEqual(eRSRFields, rsrFlds) { t.Errorf("Expecting: %v, received: %v", eRSRFields, rsrFlds) } + fields := `host,~sip_redirected_to:s/sip:\+49(\d+)@/0$1/,destination` + expectParsedFields := []*RSRField{ + &RSRField{Id: "host"}, + &RSRField{Id: "sip_redirected_to", + RSRules: []*ReSearchReplace{&ReSearchReplace{SearchRegexp: regexp.MustCompile(`sip:\+49(\d+)@`), ReplaceTemplate: "0$1"}}}, + &RSRField{Id: "destination"}} + if parsedFields, err := ParseRSRFields(fields, FIELDS_SEP); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if !reflect.DeepEqual(parsedFields, expectParsedFields) { + t.Errorf("Unexpected value of parsed fields") + } } From d47e1bf53b522a0f4a1c0cd412573010ea95c514 Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 11 Jul 2014 10:28:01 +0200 Subject: [PATCH 19/25] DerivedCharging run filters are now chained --- cdrc/cdrc.go | 9 ++++--- config/helpers.go | 5 +--- config/helpers_test.go | 38 ++++++++++++++++++++++++++++++ mediator/mediator.go | 8 ++++++- sessionmanager/fssessionmanager.go | 7 +++++- utils/consts.go | 1 + utils/rsrfield_test.go | 10 ++++++++ utils/storedcdr_test.go | 18 ++++++++++++++ 8 files changed, 87 insertions(+), 9 deletions(-) diff --git a/cdrc/cdrc.go b/cdrc/cdrc.go index c9b641092..e3a9ead15 100644 --- a/cdrc/cdrc.go +++ b/cdrc/cdrc.go @@ -197,12 +197,14 @@ func (self *Cdrc) processFile(filePath string) error { csvReader := csv.NewReader(bufio.NewReader(file)) csvReader.Comma = self.csvSep procRowNr := 0 + timeStart := time.Now() for { - procRowNr += 1 record, err := csvReader.Read() if err != nil && err == io.EOF { break // End of file - } else if err != nil { + } + procRowNr += 1 // Only increase if not end of file + if err != nil { engine.Logger.Err(fmt.Sprintf(" Error in csv file: %s", err.Error())) continue // Other csv related errors, ignore } @@ -229,6 +231,7 @@ func (self *Cdrc) processFile(filePath string) error { engine.Logger.Err(err.Error()) return err } - engine.Logger.Info(fmt.Sprintf("Finished processing %s, moved to %s", fn, newPath)) + engine.Logger.Info(fmt.Sprintf("Finished processing %s, moved to %s. Total records processed: %d, run duration: %s", + fn, newPath, procRowNr, time.Now().Sub(timeStart))) return nil } diff --git a/config/helpers.go b/config/helpers.go index b4c061f97..53522e241 100644 --- a/config/helpers.go +++ b/config/helpers.go @@ -29,11 +29,8 @@ import ( // Adds support for slice values in config func ConfigSlice(cfgVal string) ([]string, error) { - cfgValStrs := strings.Split(cfgVal, ",") // If need arrises, we can make the separator configurable + cfgValStrs := strings.Split(cfgVal, utils.FIELDS_SEP) // If need arrises, we can make the separator configurable for idx, elm := range cfgValStrs { - //if elm == "" { //One empty element is presented when splitting empty string - // return nil, errors.New("Empty values in config slice") - //} cfgValStrs[idx] = strings.TrimSpace(elm) // By default spaces are not removed so we do it here to avoid unpredicted results in config } return cfgValStrs, nil diff --git a/config/helpers_test.go b/config/helpers_test.go index a64d02f52..11f0177bc 100644 --- a/config/helpers_test.go +++ b/config/helpers_test.go @@ -19,6 +19,7 @@ along with this program. If not, see package config import ( + "encoding/json" "reflect" "testing" @@ -61,6 +62,43 @@ usage_fields = test1, test2 } } +func TestParseCfgDerivedChargingDn1(t *testing.T) { + eFieldsCfg := []byte(`[derived_charging] +run_ids = run1, run2 +run_filters =~account:s/^\w+[mpls]\d{6}$//,~account:s/^0\d{9}$//;^account/value/ +reqtype_fields = test1, test2 +direction_fields = test1, test2 +tenant_fields = test1, test2 +category_fields = test1, test2 +account_fields = test1, test2 +subject_fields = test1, test2 +destination_fields = test1, test2 +setup_time_fields = test1, test2 +answer_time_fields = test1, test2 +usage_fields = test1, test2 +`) + eDcs := make(utils.DerivedChargers, 2) + if dc, err := utils.NewDerivedCharger("run1", `~account:s/^\w+[mpls]\d{6}$//`, "test1", "test1", "test1", + "test1", "test1", "test1", "test1", "test1", "test1", "test1"); err != nil { + t.Error("Unexpected error: ", err) + } else { + eDcs[0] = dc + } + if dc, err := utils.NewDerivedCharger("run2", `~account:s/^0\d{9}$//;^account/value/`, "test2", "test2", "test2", + "test2", "test2", "test2", "test2", "test2", "test2", "test2"); err != nil { + t.Error("Unexpected error: ", err) + } else { + eDcs[1] = dc + } + + if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil { + t.Error("Could not parse the config", err.Error()) + } else if !reflect.DeepEqual(cfg.DerivedChargers, eDcs) { + dcsJson, _ := json.Marshal(cfg.DerivedChargers) + t.Errorf("Received: %s", string(dcsJson)) + } +} + func TestParseCdrcCdrFields(t *testing.T) { eFieldsCfg := []byte(`[cdrc] cdr_type = test diff --git a/mediator/mediator.go b/mediator/mediator.go index d2005dfcb..9dbbe88d9 100644 --- a/mediator/mediator.go +++ b/mediator/mediator.go @@ -121,11 +121,16 @@ func (self *Mediator) RateCdr(storedCdr *utils.StoredCdr) error { } for _, dc := range dcs { runFilters, _ := utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP) + matchingAllFilters := true for _, dcRunFilter := range runFilters { if fltrPass, _ := storedCdr.PassesFieldFilter(dcRunFilter); !fltrPass { - continue + matchingAllFilters = false + break } } + if !matchingAllFilters { // Do not process the derived charger further if not all filters were matched + continue + } dcReqTypeFld, _ := utils.NewRSRField(dc.ReqTypeField) dcDirFld, _ := utils.NewRSRField(dc.DirectionField) dcTenantFld, _ := utils.NewRSRField(dc.TenantField) @@ -143,6 +148,7 @@ func (self *Mediator) RateCdr(storedCdr *utils.StoredCdr) error { err.Error()) // Cannot fork CDR, important just runid and error continue } + engine.Logger.Debug(fmt.Sprintf("Appending CdrRun: %+v\n", forkedCdr)) cdrRuns = append(cdrRuns, forkedCdr) } for _, cdr := range cdrRuns { diff --git a/sessionmanager/fssessionmanager.go b/sessionmanager/fssessionmanager.go index 341791f2f..29fe30bf6 100644 --- a/sessionmanager/fssessionmanager.go +++ b/sessionmanager/fssessionmanager.go @@ -162,11 +162,16 @@ func (sm *FSSessionManager) OnChannelPark(ev Event) { dcs, _ = dcs.AppendDefaultRun() for _, dc := range dcs { runFilters, _ := utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP) + matchingAllFilters := true for _, dcRunFilter := range runFilters { if fltrPass, _ := ev.PassesFieldFilter(dcRunFilter); !fltrPass { - continue + matchingAllFilters = false + break } } + if !matchingAllFilters { // Do not process the derived charger further if not all filters were matched + continue + } startTime, err := ev.GetAnswerTime(PARK_TIME) if err != nil { engine.Logger.Err("Error parsing answer event start time, using time.Now!") diff --git a/utils/consts.go b/utils/consts.go index 014fba60a..f410e311a 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -74,6 +74,7 @@ const ( FIELDS_SEP = "," REGEXP_PREFIX = "~" JSON = "json" + GOB = "gob" MSGPACK = "msgpack" CSV_LOAD = "CSVLOAD" CGRID = "cgrid" diff --git a/utils/rsrfield_test.go b/utils/rsrfield_test.go index 9296da4dc..2eca3667d 100644 --- a/utils/rsrfield_test.go +++ b/utils/rsrfield_test.go @@ -165,3 +165,13 @@ func TestParseRSRFields(t *testing.T) { t.Errorf("Unexpected value of parsed fields") } } + +func TestParseCdrcDn1(t *testing.T) { + if rl, err := NewRSRField(`~1:s/^00(\d+)(?:[a-zA-Z].{3})*0*([1-9]\d+)$/+$1$2/:s/^\+49(18\d{2})$/+491400$1/`); err != nil { + t.Error("Unexpected error: ", err) + } else if parsed := rl.ParseValue("0049ABOC0630415354"); parsed != "+49630415354" { + t.Errorf("Expecting: +49630415354, received: %s", parsed) + } else if parsed2 := rl.ParseValue("00491888"); parsed2 != "+4914001888" { + t.Errorf("Expecting: +4914001888, received: %s", parsed2) + } +} diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index bfc02bfd9..62a3bf97c 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -130,6 +130,7 @@ func TestPassesFieldFilterDn1(t *testing.T) { if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass { t.Error("Not passing valid filter") } + cdr = &StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "futurem00005", ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, } @@ -153,6 +154,23 @@ func TestPassesFieldFilterDn1(t *testing.T) { if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { t.Error("Should not pass filter") } + cdr = &StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), Account: "0162447222", + ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + } + if acntPrefxFltr, err := NewRSRField(`~account:s/^0\d{9}$//`); err != nil { + t.Error("Unexpected parse error", err) + } else if acntPrefxFltr == nil { + t.Error("Failed parsing rule") + } else if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); !pass { + t.Error("Not passing valid filter") + } + if acntPrefxFltr, err := NewRSRField(`~account:s/^\w+[shmp]\d{4}$//`); err != nil { + t.Error("Unexpected parse error", err) + } else if acntPrefxFltr == nil { + t.Error("Failed parsing rule") + } else if pass, _ := cdr.PassesFieldFilter(acntPrefxFltr); pass { + t.Error("Should not pass filter") + } } func TestUsageMultiply(t *testing.T) { From 192ce121de821e9a07222194fca0bdc85754deda Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 11 Jul 2014 16:40:38 +0200 Subject: [PATCH 20/25] Adding re-connecting rpc client --- cmd/cgr-engine/cgr-engine.go | 10 +++++----- engine/responder.go | 3 ++- mediator/mediator.go | 4 +++- update_external_libs.sh | 1 + 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 9186762cc..118078e34 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -23,7 +23,6 @@ import ( "flag" "fmt" "log" - "net/rpc" "os" //"runtime" "strconv" @@ -40,6 +39,7 @@ import ( "github.com/cgrates/cgrates/scheduler" "github.com/cgrates/cgrates/sessionmanager" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" ) const ( @@ -93,11 +93,11 @@ func startMediator(responder *engine.Responder, loggerDb engine.LogStorage, cdrD <-cacheChan // Cache needs to come up before we are ready connector = responder } else { - var client *rpc.Client + var client *rpcclient.RpcClient var err error for i := 0; i < cfg.MediatorRaterReconnects; i++ { - client, err = rpc.Dial("tcp", cfg.MediatorRater) + client, err = rpcclient.NewRpcClient("tcp", cfg.MediatorRater, 0, 3, utils.GOB) if err == nil { //Connected so no need to reiterate break } @@ -146,11 +146,11 @@ func startSessionManager(responder *engine.Responder, loggerDb engine.LogStorage <-cacheChan // Wait for the cache to init before start doing queries connector = responder } else { - var client *rpc.Client + var client *rpcclient.RpcClient var err error for i := 0; i < cfg.SMRaterReconnects; i++ { - client, err = rpc.Dial("tcp", cfg.SMRater) + client, err = rpcclient.NewRpcClient("tcp", cfg.SMRater, 0, 3, utils.GOB) if err == nil { //Connected so no need to reiterate break } diff --git a/engine/responder.go b/engine/responder.go index 176a0aa69..d0b879411 100644 --- a/engine/responder.go +++ b/engine/responder.go @@ -30,6 +30,7 @@ import ( "github.com/cgrates/cgrates/balancer2go" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/utils" + "github.com/cgrates/rpcclient" ) type Responder struct { @@ -289,7 +290,7 @@ type Connector interface { } type RPCClientConnector struct { - Client *rpc.Client + Client *rpcclient.RpcClient } func (rcc *RPCClientConnector) GetCost(cd CallDescriptor, cc *CallCost) error { diff --git a/mediator/mediator.go b/mediator/mediator.go index 9dbbe88d9..6e133a10d 100644 --- a/mediator/mediator.go +++ b/mediator/mediator.go @@ -121,9 +121,12 @@ func (self *Mediator) RateCdr(storedCdr *utils.StoredCdr) error { } for _, dc := range dcs { runFilters, _ := utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP) + engine.Logger.Debug(fmt.Sprintf("RunFiltersStr: %s", dc.RunFilters)) matchingAllFilters := true for _, dcRunFilter := range runFilters { + engine.Logger.Debug(fmt.Sprintf("Processing runFilter Id: %s with rules: %+v", dcRunFilter.Id, dcRunFilter.RSRules[0])) if fltrPass, _ := storedCdr.PassesFieldFilter(dcRunFilter); !fltrPass { + engine.Logger.Debug(fmt.Sprintf("Not matching runFilter Id: %s with rules: %+v, cdr: %+v", dcRunFilter.Id, dcRunFilter.RSRules[0], storedCdr)) matchingAllFilters = false break } @@ -148,7 +151,6 @@ func (self *Mediator) RateCdr(storedCdr *utils.StoredCdr) error { err.Error()) // Cannot fork CDR, important just runid and error continue } - engine.Logger.Debug(fmt.Sprintf("Appending CdrRun: %+v\n", forkedCdr)) cdrRuns = append(cdrRuns, forkedCdr) } for _, cdr := range cdrRuns { diff --git a/update_external_libs.sh b/update_external_libs.sh index 3303f1082..5d9939702 100755 --- a/update_external_libs.sh +++ b/update_external_libs.sh @@ -8,3 +8,4 @@ go get -u -v github.com/go-sql-driver/mysql go get -u -v github.com/hoisie/redis go get -u -v github.com/howeyc/fsnotify go get -u -v github.com/cgrates/liner +go get -u -v github.com/cgrates/rpcclient From c8c52842cb7c6c3aac28e2715be6267375f755bd Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 11 Jul 2014 17:00:19 +0200 Subject: [PATCH 21/25] Removing debug log --- mediator/mediator.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/mediator/mediator.go b/mediator/mediator.go index 6e133a10d..14757881d 100644 --- a/mediator/mediator.go +++ b/mediator/mediator.go @@ -121,12 +121,9 @@ func (self *Mediator) RateCdr(storedCdr *utils.StoredCdr) error { } for _, dc := range dcs { runFilters, _ := utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP) - engine.Logger.Debug(fmt.Sprintf("RunFiltersStr: %s", dc.RunFilters)) matchingAllFilters := true for _, dcRunFilter := range runFilters { - engine.Logger.Debug(fmt.Sprintf("Processing runFilter Id: %s with rules: %+v", dcRunFilter.Id, dcRunFilter.RSRules[0])) if fltrPass, _ := storedCdr.PassesFieldFilter(dcRunFilter); !fltrPass { - engine.Logger.Debug(fmt.Sprintf("Not matching runFilter Id: %s with rules: %+v, cdr: %+v", dcRunFilter.Id, dcRunFilter.RSRules[0], storedCdr)) matchingAllFilters = false break } From 31a573fde4bb252ebb87ec5f326d5f096093a05b Mon Sep 17 00:00:00 2001 From: DanB Date: Fri, 11 Jul 2014 18:22:50 +0200 Subject: [PATCH 22/25] =?UTF-8?q?=C2RSRField=20parser=20to=20accept=20one?= =?UTF-8?q?=20letter=20replace=20templates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/cgr-engine/cgr-engine.go | 4 ++-- utils/rsrfield.go | 8 ++++---- utils/rsrfield_test.go | 3 +++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/cgr-engine/cgr-engine.go b/cmd/cgr-engine/cgr-engine.go index 118078e34..c91c0bce3 100644 --- a/cmd/cgr-engine/cgr-engine.go +++ b/cmd/cgr-engine/cgr-engine.go @@ -97,7 +97,7 @@ func startMediator(responder *engine.Responder, loggerDb engine.LogStorage, cdrD var err error for i := 0; i < cfg.MediatorRaterReconnects; i++ { - client, err = rpcclient.NewRpcClient("tcp", cfg.MediatorRater, 0, 3, utils.GOB) + client, err = rpcclient.NewRpcClient("tcp", cfg.MediatorRater, 0, cfg.MediatorRaterReconnects, utils.GOB) if err == nil { //Connected so no need to reiterate break } @@ -150,7 +150,7 @@ func startSessionManager(responder *engine.Responder, loggerDb engine.LogStorage var err error for i := 0; i < cfg.SMRaterReconnects; i++ { - client, err = rpcclient.NewRpcClient("tcp", cfg.SMRater, 0, 3, utils.GOB) + client, err = rpcclient.NewRpcClient("tcp", cfg.SMRater, 0, cfg.SMRaterReconnects, utils.GOB) if err == nil { //Connected so no need to reiterate break } diff --git a/utils/rsrfield.go b/utils/rsrfield.go index 20ceaaac5..4f4fa604b 100644 --- a/utils/rsrfield.go +++ b/utils/rsrfield.go @@ -48,17 +48,17 @@ func NewRSRField(fldStr string) (*RSRField, error) { spltRgxp := regexp.MustCompile(`:s\/`) spltRules := spltRgxp.Split(fldStr, -1) if len(spltRules) < 2 { - return nil, fmt.Errorf("Invalid Search&Replace field rule. %s", fldStr) + return nil, fmt.Errorf("Invalid Split of Search&Replace field rule. %s", fldStr) } rsrField := &RSRField{Id: spltRules[0][1:]} // Original id in form ~hdr_name - rulesRgxp := regexp.MustCompile(`(?:(.+[^\\])\/(.+[^\\])*\/){1,}`) + rulesRgxp := regexp.MustCompile(`(?:(.+[^\\])\/(.*[^\\])*\/){1,}`) for _, ruleStr := range spltRules[1:] { // :s/ already removed through split allMatches := rulesRgxp.FindStringSubmatch(ruleStr) if len(allMatches) != 3 { - return nil, fmt.Errorf("Invalid Search&Replace field rule. %s", fldStr) + return nil, fmt.Errorf("Not enough members in Search&Replace, ruleStr: %s, matches: %v, ", ruleStr, allMatches) } if srRegexp, err := regexp.Compile(allMatches[1]); err != nil { - return nil, fmt.Errorf("Invalid Search&Replace field rule. %s", fldStr) + return nil, fmt.Errorf("Invalid Search&Replace subfield rule: %s", allMatches[1]) } else { rsrField.RSRules = append(rsrField.RSRules, &ReSearchReplace{SearchRegexp: srRegexp, ReplaceTemplate: allMatches[2]}) } diff --git a/utils/rsrfield_test.go b/utils/rsrfield_test.go index 2eca3667d..6b9339aab 100644 --- a/utils/rsrfield_test.go +++ b/utils/rsrfield_test.go @@ -65,6 +65,9 @@ func TestNewRSRFieldIvo(t *testing.T) { } else if !reflect.DeepEqual(rsrField, expectRSRField) { t.Errorf("Unexpected RSRField received: %v", rsrField) } + if _, err := NewRSRField(`~account:s/^[A-Za-z0-9]*[c|a]\d{4}$/S/:s/^[A-Za-z0-9]*n\d{4}$/C/:s/^\d{10}$//`); err != nil { + t.Error(err) + } } func TestConvertPlusNationalAnd00(t *testing.T) { From 430da9b714472ddf19f8cd88b24bc96211c94b46 Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 15 Jul 2014 11:59:46 +0200 Subject: [PATCH 23/25] CdrExporter does not longer make differences between .csv and .fwv formats handling, treating fields mandatory property also --- apier/cdre.go | 3 +- cdre/cdrexporter.go | 30 +++---- cdre/libcdre.go | 3 + config/cdreconfig.go | 178 +++++++++++++++++++++----------------- config/cdreconfig_test.go | 86 +++++------------- config/config.go | 3 +- config/config_test.go | 20 ++--- 7 files changed, 146 insertions(+), 177 deletions(-) diff --git a/apier/cdre.go b/apier/cdre.go index 7018621eb..3628aa67d 100644 --- a/apier/cdre.go +++ b/apier/cdre.go @@ -59,7 +59,8 @@ func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.E } } else { exportTemplate, _ = config.NewDefaultCdreConfig() - if contentFlds, err := config.NewCdreCdrFieldsFromIds(strings.Split(*attr.ExportTemplate, string(utils.CSV_SEP))...); err != nil { + if contentFlds, err := config.NewCdreCdrFieldsFromIds(exportTemplate.CdrFormat == utils.CDRE_FIXED_WIDTH, + strings.Split(*attr.ExportTemplate, string(utils.CSV_SEP))...); err != nil { return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error()) } else { exportTemplate.ContentFields = contentFlds diff --git a/cdre/cdrexporter.go b/cdre/cdrexporter.go index ef6201a99..842bb7deb 100644 --- a/cdre/cdrexporter.go +++ b/cdre/cdrexporter.go @@ -241,11 +241,9 @@ func (cdre *CdrExporter) composeHeader() error { return err } fmtOut := outVal - if cdre.cdrFormat == utils.CDRE_FIXED_WIDTH { - if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR header, field %s, error: %s", cfgFld.Name, err.Error())) - return err - } + if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR header, field %s, error: %s", cfgFld.Name, err.Error())) + return err } cdre.header = append(cdre.header, fmtOut) } @@ -272,11 +270,9 @@ func (cdre *CdrExporter) composeTrailer() error { return err } fmtOut := outVal - if cdre.cdrFormat == utils.CDRE_FIXED_WIDTH { - if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR trailer, field: %s, error: %s", cfgFld.Name, err.Error())) - return err - } + if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR trailer, field: %s, error: %s", cfgFld.Name, err.Error())) + return err } cdre.trailer = append(cdre.trailer, fmtOut) } @@ -338,11 +334,9 @@ func (cdre *CdrExporter) processCdr(cdr *utils.StoredCdr) error { return err } fmtOut := outVal - if cdre.cdrFormat == utils.CDRE_FIXED_WIDTH { - if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { - engine.Logger.Err(fmt.Sprintf(" Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Name, outVal, err.Error())) - return err - } + if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil { + engine.Logger.Err(fmt.Sprintf(" Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Name, outVal, err.Error())) + return err } cdrRow[idx] += fmtOut } @@ -362,8 +356,10 @@ func (cdre *CdrExporter) processCdr(cdr *utils.StoredCdr) error { if !utils.IsSliceMember([]string{utils.DATA, utils.SMS}, cdr.TOR) { // Only count duration for non data cdrs cdre.totalDuration += cdr.Usage } - cdre.totalCost += cdr.Cost - cdre.totalCost = utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE) + if cdr.Cost != -1 { + cdre.totalCost += cdr.Cost + cdre.totalCost = utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE) + } if cdre.firstExpOrderId > cdr.OrderId || cdre.firstExpOrderId == 0 { cdre.firstExpOrderId = cdr.OrderId } diff --git a/cdre/libcdre.go b/cdre/libcdre.go index 38d507b7c..d1569720d 100644 --- a/cdre/libcdre.go +++ b/cdre/libcdre.go @@ -35,6 +35,9 @@ func FmtFieldWidth(source string, width int, strip, padding string, mandatory bo if mandatory && len(source) == 0 { return "", errors.New("Empty source value") } + if width == 0 { // Disable width processing if not defined + return source, nil + } if len(source) == width { // the source is exactly the maximum length return source, nil } diff --git a/config/cdreconfig.go b/config/cdreconfig.go index 7fea319d0..1df24a9fe 100644 --- a/config/cdreconfig.go +++ b/config/cdreconfig.go @@ -24,15 +24,16 @@ import ( ) // Converts a list of field identifiers into proper CDR field content -func NewCdreCdrFieldsFromIds(fldsIds ...string) ([]*CdreCdrField, error) { +func NewCdreCdrFieldsFromIds(withFixedWith bool, fldsIds ...string) ([]*CdreCdrField, error) { cdrFields := make([]*CdreCdrField, len(fldsIds)) for idx, fldId := range fldsIds { if parsedRsr, err := utils.NewRSRField(fldId); err != nil { return nil, err } else { cdrFld := &CdreCdrField{Name: fldId, Type: utils.CDRFIELD, Value: fldId, valueAsRsrField: parsedRsr} - if err := cdrFld.setDefaultFixedWidthProperties(); err != nil { // Set default fixed width properties to be used later if needed + if err := cdrFld.setDefaultFieldProperties(withFixedWith); err != nil { // Set default fixed width properties to be used later if needed return nil, err + } cdrFields[idx] = cdrFld } @@ -73,7 +74,7 @@ func (cdreCfg *CdreConfig) setDefaults() error { cdreCfg.MaskDestId = "" cdreCfg.MaskLength = 0 cdreCfg.ExportDir = "/var/log/cgrates/cdre" - if flds, err := NewCdreCdrFieldsFromIds(utils.CGRID, utils.MEDI_RUNID, utils.TOR, utils.ACCID, utils.REQTYPE, utils.DIRECTION, utils.TENANT, + if flds, err := NewCdreCdrFieldsFromIds(false, utils.CGRID, utils.MEDI_RUNID, utils.TOR, utils.ACCID, utils.REQTYPE, utils.DIRECTION, utils.TENANT, utils.CATEGORY, utils.ACCOUNT, utils.SUBJECT, utils.DESTINATION, utils.SETUP_TIME, utils.ANSWER_TIME, utils.USAGE, utils.COST); err != nil { return err } else { @@ -100,125 +101,142 @@ func (cdrField *CdreCdrField) ValueAsRSRField() *utils.RSRField { } // Should be called on .fwv configuration without providing default values for fixed with parameters -func (cdrField *CdreCdrField) setDefaultFixedWidthProperties() error { +func (cdrField *CdreCdrField) setDefaultFieldProperties(fixedWidth bool) error { if cdrField.valueAsRsrField == nil { return errors.New("Missing valueAsRsrField") } switch cdrField.valueAsRsrField.Id { case utils.CGRID: - cdrField.Width = 40 - cdrField.Strip = "" - cdrField.Padding = "" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 40 + } case utils.ORDERID: - cdrField.Width = 11 - cdrField.Strip = "" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 11 + cdrField.Padding = "left" + } case utils.TOR: - cdrField.Width = 6 - cdrField.Strip = "" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 6 + cdrField.Padding = "left" + } case utils.ACCID: - cdrField.Width = 36 - cdrField.Strip = "left" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 36 + cdrField.Strip = "left" + cdrField.Padding = "left" + } case utils.CDRHOST: - cdrField.Width = 15 - cdrField.Strip = "left" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 15 + cdrField.Strip = "left" + cdrField.Padding = "left" + } case utils.CDRSOURCE: - cdrField.Width = 15 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 15 + cdrField.Strip = "xright" + cdrField.Padding = "left" + } case utils.REQTYPE: - cdrField.Width = 13 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 13 + cdrField.Strip = "xright" + cdrField.Padding = "left" + } case utils.DIRECTION: - cdrField.Width = 4 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 4 + cdrField.Strip = "xright" + cdrField.Padding = "left" + } case utils.TENANT: - cdrField.Width = 24 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 24 + cdrField.Strip = "xright" + cdrField.Padding = "left" + } case utils.CATEGORY: - cdrField.Width = 10 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 10 + cdrField.Strip = "xright" + cdrField.Padding = "left" + } case utils.ACCOUNT: - cdrField.Width = 24 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 24 + cdrField.Strip = "xright" + cdrField.Padding = "left" + } case utils.SUBJECT: - cdrField.Width = 24 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 24 + cdrField.Strip = "xright" + cdrField.Padding = "left" + } case utils.DESTINATION: - cdrField.Width = 24 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 24 + cdrField.Strip = "xright" + cdrField.Padding = "left" + } case utils.SETUP_TIME: - cdrField.Width = 30 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "2006-01-02T15:04:05Z07:00" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 30 + cdrField.Strip = "xright" + cdrField.Padding = "left" + cdrField.Layout = "2006-01-02T15:04:05Z07:00" + } case utils.ANSWER_TIME: - cdrField.Width = 30 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "2006-01-02T15:04:05Z07:00" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 30 + cdrField.Strip = "xright" + cdrField.Padding = "left" + cdrField.Layout = "2006-01-02T15:04:05Z07:00" + } case utils.USAGE: - cdrField.Width = 30 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 30 + cdrField.Strip = "xright" + cdrField.Padding = "left" + } case utils.MEDI_RUNID: - cdrField.Width = 20 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 20 + cdrField.Strip = "xright" + cdrField.Padding = "left" + } case utils.COST: - cdrField.Width = 24 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = true + if fixedWidth { + cdrField.Width = 24 + cdrField.Strip = "xright" + cdrField.Padding = "left" + } default: - cdrField.Width = 30 - cdrField.Strip = "xright" - cdrField.Padding = "left" - cdrField.Layout = "" cdrField.Mandatory = false + if fixedWidth { + cdrField.Width = 30 + cdrField.Strip = "xright" + cdrField.Padding = "left" + } } return nil } diff --git a/config/cdreconfig_test.go b/config/cdreconfig_test.go index afbe55b76..18e65a8cd 100644 --- a/config/cdreconfig_test.go +++ b/config/cdreconfig_test.go @@ -49,7 +49,7 @@ func TestCdreCfgNewCdreCdrFieldsFromIds(t *testing.T) { valueAsRsrField: &utils.RSRField{Id: "extra1"}, }, } - if cdreFlds, err := NewCdreCdrFieldsFromIds(utils.CGRID, "extra1"); err != nil { + if cdreFlds, err := NewCdreCdrFieldsFromIds(true, utils.CGRID, "extra1"); err != nil { t.Error(err) } else if !reflect.DeepEqual(expectedFlds, cdreFlds) { t.Errorf("Expected: %v, received: %v", expectedFlds, cdreFlds) @@ -86,7 +86,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.CGRID, Type: utils.CDRFIELD, Value: utils.CGRID, - Width: 40, Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.CGRID}, }, @@ -94,9 +93,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.MEDI_RUNID, Type: utils.CDRFIELD, Value: utils.MEDI_RUNID, - Width: 20, - Strip: "xright", - Padding: "left", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.MEDI_RUNID}, }, @@ -104,8 +100,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.TOR, Type: utils.CDRFIELD, Value: utils.TOR, - Width: 6, - Padding: "left", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.TOR}, }, @@ -113,9 +107,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.ACCID, Type: utils.CDRFIELD, Value: utils.ACCID, - Width: 36, - Strip: "left", - Padding: "left", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.ACCID}, }, @@ -123,9 +114,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.REQTYPE, Type: utils.CDRFIELD, Value: utils.REQTYPE, - Width: 13, - Strip: "xright", - Padding: "left", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.REQTYPE}, }, @@ -133,9 +121,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.DIRECTION, Type: utils.CDRFIELD, Value: utils.DIRECTION, - Width: 4, - Strip: "xright", - Padding: "left", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.DIRECTION}, }, @@ -143,9 +128,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.TENANT, Type: utils.CDRFIELD, Value: utils.TENANT, - Width: 24, - Strip: "xright", - Padding: "left", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.TENANT}, }, @@ -153,9 +135,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.CATEGORY, Type: utils.CDRFIELD, Value: utils.CATEGORY, - Width: 10, - Strip: "xright", - Padding: "left", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.CATEGORY}, }, @@ -163,9 +142,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.ACCOUNT, Type: utils.CDRFIELD, Value: utils.ACCOUNT, - Width: 24, - Strip: "xright", - Padding: "left", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.ACCOUNT}, }, @@ -173,9 +149,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.SUBJECT, Type: utils.CDRFIELD, Value: utils.SUBJECT, - Width: 24, - Strip: "xright", - Padding: "left", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.SUBJECT}, }, @@ -183,9 +156,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.DESTINATION, Type: utils.CDRFIELD, Value: utils.DESTINATION, - Width: 24, - Strip: "xright", - Padding: "left", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.DESTINATION}, }, @@ -193,10 +163,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.SETUP_TIME, Type: utils.CDRFIELD, Value: utils.SETUP_TIME, - Width: 30, - Strip: "xright", - Padding: "left", - Layout: "2006-01-02T15:04:05Z07:00", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.SETUP_TIME}, }, @@ -204,10 +170,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.ANSWER_TIME, Type: utils.CDRFIELD, Value: utils.ANSWER_TIME, - Width: 30, - Strip: "xright", - Padding: "left", - Layout: "2006-01-02T15:04:05Z07:00", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.ANSWER_TIME}, }, @@ -215,9 +177,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.USAGE, Type: utils.CDRFIELD, Value: utils.USAGE, - Width: 30, - Strip: "xright", - Padding: "left", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.USAGE}, }, @@ -225,9 +184,6 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { Name: utils.COST, Type: utils.CDRFIELD, Value: utils.COST, - Width: 24, - Strip: "xright", - Padding: "left", Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.COST}, }, @@ -239,7 +195,7 @@ func TestCdreCfgNewDefaultCdreConfig(t *testing.T) { } } -func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { +func TestCdreCfgSetDefaultFieldProperties(t *testing.T) { cdreCdrFld := &CdreCdrField{ valueAsRsrField: &utils.RSRField{Id: utils.CGRID}, } @@ -248,7 +204,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.CGRID}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -262,7 +218,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.ORDERID}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -276,7 +232,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.TOR}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -291,7 +247,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.ACCID}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -306,7 +262,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.CDRHOST}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -321,7 +277,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.CDRSOURCE}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -336,7 +292,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.REQTYPE}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -351,7 +307,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.DIRECTION}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -366,7 +322,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.TENANT}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -381,7 +337,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.CATEGORY}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -396,7 +352,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.ACCOUNT}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -411,7 +367,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.SUBJECT}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -426,7 +382,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.DESTINATION}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -442,7 +398,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.SETUP_TIME}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -458,7 +414,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.ANSWER_TIME}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -473,7 +429,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.USAGE}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -488,7 +444,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.MEDI_RUNID}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -503,7 +459,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: true, valueAsRsrField: &utils.RSRField{Id: utils.COST}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) @@ -518,7 +474,7 @@ func TestCdreCfgSetDefaultFixedWidthProperties(t *testing.T) { Mandatory: false, valueAsRsrField: &utils.RSRField{Id: "extra_1"}, } - if err := cdreCdrFld.setDefaultFixedWidthProperties(); err != nil { + if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil { t.Error(err) } else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) { t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld) diff --git a/config/config.go b/config/config.go index 7e7afd546..5e2e1b633 100644 --- a/config/config.go +++ b/config/config.go @@ -423,7 +423,8 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { cfg.CdreDefaultInstance = xmlTemplates[exportTemplate[len(utils.XML_PROFILE_PREFIX):]].AsCdreConfig() } } else { // Not loading out of template - if flds, err := NewCdreCdrFieldsFromIds(strings.Split(exportTemplate, string(utils.CSV_SEP))...); err != nil { + if flds, err := NewCdreCdrFieldsFromIds(cfg.CdreDefaultInstance.CdrFormat == utils.CDRE_FIXED_WIDTH, + strings.Split(exportTemplate, string(utils.CSV_SEP))...); err != nil { return nil, err } else { cfg.CdreDefaultInstance.ContentFields = flds diff --git a/config/config_test.go b/config/config_test.go index 3bf32117b..528f9fd76 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -214,7 +214,7 @@ func TestConfigFromFile(t *testing.T) { MaskDestId: "test", MaskLength: 99, ExportDir: "test"} - eCfg.CdreDefaultInstance.ContentFields, _ = NewCdreCdrFieldsFromIds("test") + eCfg.CdreDefaultInstance.ContentFields, _ = NewCdreCdrFieldsFromIds(false, "test") eCfg.CdrcEnabled = true eCfg.CdrcCdrs = "test" eCfg.CdrcRunDelay = time.Duration(99) * time.Second @@ -264,13 +264,7 @@ func TestConfigFromFile(t *testing.T) { eCfg.MailerFromAddr = "test" if !reflect.DeepEqual(cfg, eCfg) { t.Log(eCfg) - for _, eDC := range eCfg.DerivedChargers { - fmt.Printf("ExpectDerivedChargers: %+v\n", eDC) - } t.Log(cfg) - for _, eDC := range cfg.DerivedChargers { - fmt.Printf("DerivedChargers: %+v\n", eDC) - } t.Error("Loading of configuration from file failed!") } } @@ -308,11 +302,11 @@ cdr_format = csv export_template = cgrid,mediation_runid,accid `) expectedFlds := []*CdreCdrField{ - &CdreCdrField{Name: "cgrid", Type: utils.CDRFIELD, Value: "cgrid", valueAsRsrField: &utils.RSRField{Id: "cgrid"}, Width: 40, Mandatory: true}, + &CdreCdrField{Name: "cgrid", Type: utils.CDRFIELD, Value: "cgrid", valueAsRsrField: &utils.RSRField{Id: "cgrid"}, Mandatory: true}, &CdreCdrField{Name: "mediation_runid", Type: utils.CDRFIELD, Value: "mediation_runid", valueAsRsrField: &utils.RSRField{Id: "mediation_runid"}, - Width: 20, Strip: "xright", Padding: "left", Mandatory: true}, + Mandatory: true}, &CdreCdrField{Name: "accid", Type: utils.CDRFIELD, Value: "accid", valueAsRsrField: &utils.RSRField{Id: "accid"}, - Width: 36, Strip: "left", Padding: "left", Mandatory: true}, + Mandatory: true}, } expCdreCfg := &CdreConfig{CdrFormat: utils.CSV, CostRoundingDecimals: -1, ExportDir: "/var/log/cgrates/cdre", ContentFields: expectedFlds} if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil { @@ -326,9 +320,9 @@ export_template = cgrid,~effective_caller_id_number:s/(\d+)/+$1/ `) rsrField, _ := utils.NewRSRField(`~effective_caller_id_number:s/(\d+)/+$1/`) expectedFlds = []*CdreCdrField{ - &CdreCdrField{Name: "cgrid", Type: utils.CDRFIELD, Value: "cgrid", valueAsRsrField: &utils.RSRField{Id: "cgrid"}, Width: 40, Mandatory: true}, - &CdreCdrField{Name: `~effective_caller_id_number:s/(\d+)/+$1/`, Type: utils.CDRFIELD, Value: `~effective_caller_id_number:s/(\d+)/+$1/`, valueAsRsrField: rsrField, - Width: 30, Strip: "xright", Padding: "left", Mandatory: false}} + &CdreCdrField{Name: "cgrid", Type: utils.CDRFIELD, Value: "cgrid", valueAsRsrField: &utils.RSRField{Id: "cgrid"}, Mandatory: true}, + &CdreCdrField{Name: `~effective_caller_id_number:s/(\d+)/+$1/`, Type: utils.CDRFIELD, Value: `~effective_caller_id_number:s/(\d+)/+$1/`, + valueAsRsrField: rsrField, Mandatory: false}} expCdreCfg.ContentFields = expectedFlds if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil { t.Error("Could not parse the config", err.Error()) From 8023de5f241f986731be0b5b546252ac998ffa7e Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 15 Jul 2014 16:14:58 +0300 Subject: [PATCH 24/25] action trigger balance weight and exp time also execute action trigger only on dirty balances --- engine/account.go | 11 +++++++---- engine/action_trigger.go | 25 ++++++++++++++----------- engine/actions_test.go | 6 +++--- engine/balances.go | 14 ++++++++++++++ engine/units_counter.go | 6 ++++++ 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/engine/account.go b/engine/account.go index f8eab32fe..93cb32e36 100644 --- a/engine/account.go +++ b/engine/account.go @@ -339,12 +339,12 @@ func (ub *Account) executeActionTriggers(a *Action) { if uc.BalanceType == at.BalanceType { for _, mb := range uc.Balances { if strings.Contains(at.ThresholdType, "*max") { - if mb.MatchDestination(at.DestinationId) && mb.Value >= at.ThresholdValue { + if mb.MatchActionTrigger(at) && mb.Value >= at.ThresholdValue { // run the actions at.Execute(ub) } } else { //MIN - if mb.MatchDestination(at.DestinationId) && mb.Value <= at.ThresholdValue { + if mb.MatchActionTrigger(at) && mb.Value <= at.ThresholdValue { // run the actions at.Execute(ub) } @@ -354,13 +354,16 @@ func (ub *Account) executeActionTriggers(a *Action) { } } else { // BALANCE for _, b := range ub.BalanceMap[at.BalanceType+at.Direction] { + if !b.dirty { // do not check clean balances + continue + } if strings.Contains(at.ThresholdType, "*max") { - if b.MatchDestination(at.DestinationId) && b.Value >= at.ThresholdValue { + if b.MatchActionTrigger(at) && b.Value >= at.ThresholdValue { // run the actions at.Execute(ub) } } else { //MIN - if b.MatchDestination(at.DestinationId) && b.Value <= at.ThresholdValue { + if b.MatchActionTrigger(at) && b.Value <= at.ThresholdValue { // run the actions at.Execute(ub) } diff --git a/engine/action_trigger.go b/engine/action_trigger.go index bac755d7e..418aa8ed5 100644 --- a/engine/action_trigger.go +++ b/engine/action_trigger.go @@ -22,26 +22,29 @@ import ( "encoding/json" "fmt" "sort" + "time" "github.com/cgrates/cgrates/utils" ) type ActionTrigger struct { - Id string // uniquely identify the trigger - BalanceType string - Direction string - ThresholdType string //*min_counter, *max_counter, *min_balance, *max_balance - ThresholdValue float64 - Recurrent bool // reset eexcuted flag each run - DestinationId string - Weight float64 - ActionsId string - Executed bool + Id string // uniquely identify the trigger + BalanceType string + Direction string + ThresholdType string //*min_counter, *max_counter, *min_balance, *max_balance + ThresholdValue float64 + Recurrent bool // reset eexcuted flag each run + DestinationId string + BalanceWeight float64 + BalanceExpirationDate time.Time + Weight float64 + ActionsId string + Executed bool } func (at *ActionTrigger) Execute(ub *Account) (err error) { if ub.Disabled { - return fmt.Errorf("User %s is disabled", ub.Id) + return Logger.Err(fmt.Errorf("User %s is disabled and there are triggers in action!", ub.Id)) } // does NOT need to Lock() because it is triggered from a method that took the Lock var aac Actions diff --git a/engine/actions_test.go b/engine/actions_test.go index 048844c11..6d1cda925 100644 --- a/engine/actions_test.go +++ b/engine/actions_test.go @@ -827,7 +827,7 @@ func TestActionResetCounterMinutes(t *testing.T) { CREDIT: BalanceChain{&Balance{Value: 100}}, MINUTES: BalanceChain{&Balance{Value: 10, Weight: 20, DestinationId: "NAT"}, &Balance{Weight: 10, DestinationId: "RET"}}}, UnitCounters: []*UnitsCounter{&UnitsCounter{BalanceType: CREDIT, Balances: BalanceChain{&Balance{Value: 1}}}}, - ActionTriggers: ActionTriggerPriotityList{&ActionTrigger{BalanceType: CREDIT, ThresholdValue: 2, ActionsId: "TEST_ACTIONS", Executed: true}}, + ActionTriggers: ActionTriggerPriotityList{&ActionTrigger{BalanceType: CREDIT, ThresholdType: "*max_counter", ThresholdValue: 2, ActionsId: "TEST_ACTIONS", Executed: true}}, } a := &Action{BalanceType: MINUTES} resetCounterAction(ub, a) @@ -840,14 +840,14 @@ func TestActionResetCounterMinutes(t *testing.T) { for _, b := range ub.UnitCounters[1].Balances { t.Logf("B: %+v", b) } - t.Error("Reset counters action failed!", ub.UnitCounters[1]) + t.Errorf("Reset counters action failed: %+v", ub) } if len(ub.UnitCounters) < 2 || len(ub.UnitCounters[1].Balances) < 1 { t.FailNow() } mb := ub.UnitCounters[1].Balances[0] if mb.Weight != 20 || mb.Value != 0 || mb.DestinationId != "NAT" { - t.Errorf("Balance cloned incorrectly: %v!", mb) + t.Errorf("Balance cloned incorrectly: %+v!", mb) } } diff --git a/engine/balances.go b/engine/balances.go index 53259188d..0fb946792 100644 --- a/engine/balances.go +++ b/engine/balances.go @@ -74,6 +74,20 @@ func (b *Balance) MatchDestination(destinationId string) bool { return !b.HasDestination() || b.DestinationId == destinationId } +func (b *Balance) MatchActionTrigger(at *ActionTrigger) bool { + matchesExpirationDate := true + if !at.BalanceExpirationDate.IsZero() { + matchesExpirationDate = (at.BalanceExpirationDate.Equal(b.ExpirationDate)) + } + matchesWeight := true + if at.BalanceWeight > 0 { + matchesWeight = (at.BalanceWeight == b.Weight) + } + return b.MatchDestination(at.DestinationId) && + matchesExpirationDate && + matchesWeight +} + func (b *Balance) Clone() *Balance { return &Balance{ Uuid: b.Uuid, diff --git a/engine/units_counter.go b/engine/units_counter.go index 4bc76ac12..879e8699c 100644 --- a/engine/units_counter.go +++ b/engine/units_counter.go @@ -19,6 +19,8 @@ along with this program. If not, see package engine import ( + "strings" + "github.com/cgrates/cgrates/cache2go" "github.com/cgrates/cgrates/utils" ) @@ -36,6 +38,10 @@ type UnitsCounter struct { func (uc *UnitsCounter) initBalances(ats []*ActionTrigger) { uc.Balances = BalanceChain{&Balance{}} // general balance for _, at := range ats { + if !strings.Contains(at.ThresholdType, "counter") { + // only get actions fo counter type action triggers + continue + } acs, err := accountingStorage.GetActions(at.ActionsId, false) if err != nil { continue From c44287bee078a1fc87538917c2fa7c9cdabbba36 Mon Sep 17 00:00:00 2001 From: Radu Ioan Fericean Date: Tue, 15 Jul 2014 16:16:45 +0300 Subject: [PATCH 25/25] quick fix for compile error --- engine/action_trigger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/action_trigger.go b/engine/action_trigger.go index 418aa8ed5..716423ad5 100644 --- a/engine/action_trigger.go +++ b/engine/action_trigger.go @@ -44,7 +44,7 @@ type ActionTrigger struct { func (at *ActionTrigger) Execute(ub *Account) (err error) { if ub.Disabled { - return Logger.Err(fmt.Errorf("User %s is disabled and there are triggers in action!", ub.Id)) + return fmt.Errorf("User %s is disabled and there are triggers in action!", ub.Id) } // does NOT need to Lock() because it is triggered from a method that took the Lock var aac Actions