From 31b421c7559469af38d31af8ad0b4030ef0678ce Mon Sep 17 00:00:00 2001 From: DanB Date: Mon, 7 Jul 2014 11:34:23 +0200 Subject: [PATCH] Loader for derived charging filters, using derived chargers filters within mediator and session manager, filter implementation inside StoredCdr --- .../mysql/create_tariffplan_tables.sql | 3 +- .../prepaid1centpsec/DerivedChargers.csv | 9 ++-- .../cgrates/tariffplans/DerivedChargers.csv | 4 +- engine/loader_csv.go | 42 ++++++++++--------- engine/loader_csv_test.go | 13 +++--- engine/loader_helpers.go | 4 +- engine/loader_helpers_test.go | 23 +++++----- mediator/mediator.go | 4 +- sessionmanager/fssessionmanager.go | 2 +- utils/consts.go | 2 +- utils/storedcdr.go | 11 +++++ utils/storedcdr_test.go | 26 ++++++++++++ 12 files changed, 93 insertions(+), 50 deletions(-) diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index 9557e7675..9aa46464d 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -246,7 +246,8 @@ CREATE TABLE tp_derived_chargers ( `category` varchar(16) NOT NULL, `account` varchar(24) NOT NULL, `subject` varchar(64) NOT NULL, - `runid_field` varchar(24) NOT NULL, + `run_id` varchar(24) NOT NULL, + `run_filter` varchar(24) NOT NULL, `reqtype_field` varchar(24) NOT NULL, `direction_field` varchar(24) NOT NULL, `tenant_field` varchar(24) NOT NULL, diff --git a/data/tariffplans/prepaid1centpsec/DerivedChargers.csv b/data/tariffplans/prepaid1centpsec/DerivedChargers.csv index 4b02d8f0e..7246f65fe 100644 --- a/data/tariffplans/prepaid1centpsec/DerivedChargers.csv +++ b/data/tariffplans/prepaid1centpsec/DerivedChargers.csv @@ -1,4 +1,5 @@ -#Direction,Tenant,Tor,Account,Subject,RunId,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,DurationField -*out,cgrates.org,call,dan,dan,extra1,^prepaid,,,,^rif,^rif,,,,^1s -*out,cgrates.org,call,dan,dan,extra2,,,,,^ivo,^ivo,,,, -*out,cgrates.org,call,dan,*any,extra1,,,,,^rif2,^rif2,,,, +#Direction,Tenant,Tor,Account,Subject,RunId,RunFilter,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,DurationField +*out,cgrates.org,call,dan,dan,extra1,,^prepaid,,,,^rif,^rif,,,,^1s +*out,cgrates.org,call,dan,dan,extra2,,,,,,^ivo,^ivo,,,, +*out,cgrates.org,call,dan,dan,extra3,~filterhdr1:s/(.+)/special_run3/,,,,,^runusr3,^runusr3,,,, +*out,cgrates.org,call,dan,*any,extra1,,,,,,^rif2,^rif2,,,, diff --git a/data/tutorials/fs_json/cgrates/tariffplans/DerivedChargers.csv b/data/tutorials/fs_json/cgrates/tariffplans/DerivedChargers.csv index 5a243dffd..aed05a80d 100644 --- a/data/tutorials/fs_json/cgrates/tariffplans/DerivedChargers.csv +++ b/data/tutorials/fs_json/cgrates/tariffplans/DerivedChargers.csv @@ -1,2 +1,2 @@ -#Direction,Tenant,Category,Account,Subject,RunId,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,UsageField -*out,cgrates.org,call,1001,1001,fs_json_run,^rated,*default,*default,*default,*default,^1002,*default,*default,*default,*default +#Direction,Tenant,Category,Account,Subject,RunId,RunFilter,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,UsageField +*out,cgrates.org,call,1001,1001,fs_json_run,,^rated,*default,*default,*default,*default,^1002,*default,*default,*default,*default diff --git a/engine/loader_csv.go b/engine/loader_csv.go index c44d83a58..47f2ab28f 100644 --- a/engine/loader_csv.go +++ b/engine/loader_csv.go @@ -842,16 +842,17 @@ func (csvr *CSVReader) LoadDerivedChargers() (err error) { if found { if csvr.derivedChargers[tag], err = csvr.derivedChargers[tag].Append(&utils.DerivedCharger{ RunId: ValueOrDefault(record[5], "*default"), - ReqTypeField: ValueOrDefault(record[6], "*default"), - DirectionField: ValueOrDefault(record[7], "*default"), - TenantField: ValueOrDefault(record[8], "*default"), - CategoryField: ValueOrDefault(record[9], "*default"), - AccountField: ValueOrDefault(record[10], "*default"), - SubjectField: ValueOrDefault(record[11], "*default"), - DestinationField: ValueOrDefault(record[12], "*default"), - SetupTimeField: ValueOrDefault(record[13], "*default"), - AnswerTimeField: ValueOrDefault(record[14], "*default"), - UsageField: ValueOrDefault(record[15], "*default"), + RunFilter: record[6], + ReqTypeField: ValueOrDefault(record[7], "*default"), + DirectionField: ValueOrDefault(record[8], "*default"), + TenantField: ValueOrDefault(record[9], "*default"), + CategoryField: ValueOrDefault(record[10], "*default"), + AccountField: ValueOrDefault(record[11], "*default"), + SubjectField: ValueOrDefault(record[12], "*default"), + DestinationField: ValueOrDefault(record[13], "*default"), + SetupTimeField: ValueOrDefault(record[14], "*default"), + AnswerTimeField: ValueOrDefault(record[15], "*default"), + UsageField: ValueOrDefault(record[16], "*default"), }); err != nil { return err } @@ -861,16 +862,17 @@ func (csvr *CSVReader) LoadDerivedChargers() (err error) { } csvr.derivedChargers[tag] = utils.DerivedChargers{&utils.DerivedCharger{ RunId: ValueOrDefault(record[5], "*default"), - ReqTypeField: ValueOrDefault(record[6], "*default"), - DirectionField: ValueOrDefault(record[7], "*default"), - TenantField: ValueOrDefault(record[8], "*default"), - CategoryField: ValueOrDefault(record[9], "*default"), - AccountField: ValueOrDefault(record[10], "*default"), - SubjectField: ValueOrDefault(record[11], "*default"), - DestinationField: ValueOrDefault(record[12], "*default"), - SetupTimeField: ValueOrDefault(record[13], "*default"), - AnswerTimeField: ValueOrDefault(record[14], "*default"), - UsageField: ValueOrDefault(record[15], "*default"), + RunFilter: record[6], + ReqTypeField: ValueOrDefault(record[7], "*default"), + DirectionField: ValueOrDefault(record[8], "*default"), + TenantField: ValueOrDefault(record[9], "*default"), + CategoryField: ValueOrDefault(record[10], "*default"), + AccountField: ValueOrDefault(record[11], "*default"), + SubjectField: ValueOrDefault(record[12], "*default"), + DestinationField: ValueOrDefault(record[13], "*default"), + SetupTimeField: ValueOrDefault(record[14], "*default"), + AnswerTimeField: ValueOrDefault(record[15], "*default"), + UsageField: ValueOrDefault(record[16], "*default"), }} } } diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index 63ea352dd..82e62011e 100644 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -177,10 +177,10 @@ vdf,emptyY,*out,TOPUP_EMPTY_AT, ` derivedCharges = ` -#Direction,Tenant,Category,Account,Subject,RunId,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,UsageField -*out,cgrates.org,call,dan,dan,extra1,^prepaid,,,,rif,rif,,,, -*out,cgrates.org,call,dan,dan,extra2,,,,,ivo,ivo,,,, -*out,cgrates.org,call,dan,*any,extra1,,,,,rif2,rif2,,,, +#Direction,Tenant,Category,Account,Subject,RunId,RunFilter,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,UsageField +*out,cgrates.org,call,dan,dan,extra1,^filteredHeader1/filterValue1,^prepaid,,,,rif,rif,,,, +*out,cgrates.org,call,dan,dan,extra2,,,,,,ivo,ivo,,,, +*out,cgrates.org,call,dan,*any,extra1,,,,,,rif2,rif2,,,, ` ) @@ -932,8 +932,9 @@ func TestLoadDerivedChargers(t *testing.T) { t.Error("Failed to load derivedChargers: ", csvr.derivedChargers) } expCharger1 := utils.DerivedChargers{ - &utils.DerivedCharger{RunId: "extra1", ReqTypeField: "^prepaid", DirectionField: "*default", TenantField: "*default", CategoryField: "*default", - AccountField: "rif", SubjectField: "rif", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"}, + &utils.DerivedCharger{RunId: "extra1", RunFilter: "^filteredHeader1/filterValue1", ReqTypeField: "^prepaid", DirectionField: "*default", TenantField: "*default", + CategoryField: "*default", AccountField: "rif", SubjectField: "rif", DestinationField: "*default", + SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"}, &utils.DerivedCharger{RunId: "extra2", ReqTypeField: "*default", DirectionField: "*default", TenantField: "*default", CategoryField: "*default", AccountField: "ivo", SubjectField: "ivo", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"}, } diff --git a/engine/loader_helpers.go b/engine/loader_helpers.go index 3b3b2fe44..896c2a662 100644 --- a/engine/loader_helpers.go +++ b/engine/loader_helpers.go @@ -208,8 +208,8 @@ var FileValidators = map[string]*FileLineRegexValidator{ regexp.MustCompile(`(?:\w+\s*),(?:(\w+;?)+\s*),(?:\*out\s*),(?:\w+\s*),(?:\w+\s*)$`), "Tenant([0-9A-Za-z_]),Account([0-9A-Za-z_.]),Direction(*out),ActionTimingsTag([0-9A-Za-z_]),ActionTriggersTag([0-9A-Za-z_])"}, utils.DERIVED_CHARGERS_CSV: &FileLineRegexValidator{utils.DERIVED_CHARGERS_NRCOLS, - regexp.MustCompile(`^(?:\*out),(?:[0-9A-Za-z_\.]+\s*),(?:\w+\s*),(?:\w+\s*),(?:\*any\s*|\w+\s*),(?:\w+\s*),(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?,(?:\*default\s*|\^*\w+\s*)?$`), - "Direction(*out),Tenant[0-9A-Za-z_],Category([0-9A-Za-z_]),Account[0-9A-Za-z_],Subject([0-9A-Za-z_]|*any),RunId([^0-9A-Za-z_]),ReqTypeField([^0-9A-Za-z_]|*default),DirectionField([^0-9A-Za-z_]|*default),TenantField([^0-9A-Za-z_]|*default),TorField([^0-9A-Za-z_]|*default),AccountField([^0-9A-Za-z_]|*default),SubjectField([^0-9A-Za-z_]|*default),DestinationField([^0-9A-Za-z_]|*default),SetupTimeField([^0-9A-Za-z_]|*default),AnswerTimeField([^0-9A-Za-z_]|*default),DurationField([^0-9A-Za-z_]|*default)"}, + regexp.MustCompile(`^(?:\*out),(?:[0-9A-Za-z_\.]+\s*),(?:\w+\s*),(?:\w+\s*),(?:\*any\s*|\w+\s*),(?:\w+\s*),(?:[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?,(?:\*default\s*|[~^]*[0-9A-Za-z_/:().+]+\s*)?$`), + "Direction(*out),Tenant[0-9A-Za-z_],Category([0-9A-Za-z_]),Account[0-9A-Za-z_],Subject([0-9A-Za-z_]|*any),RunId([0-9A-Za-z_]),RunFilter([^~]*[0-9A-Za-z_/]),ReqTypeField([^~]*[0-9A-Za-z_/]|*default),DirectionField([^~]*[0-9A-Za-z_/]|*default),TenantField([^~]*[0-9A-Za-z_/]|*default),TorField([^~]*[0-9A-Za-z_/]|*default),AccountField([^~]*[0-9A-Za-z_/]|*default),SubjectField([^~]*[0-9A-Za-z_/]|*default),DestinationField([^~]*[0-9A-Za-z_/]|*default),SetupTimeField([^~]*[0-9A-Za-z_/]|*default),AnswerTimeField([^~]*[0-9A-Za-z_/]|*default),DurationField([^~]*[0-9A-Za-z_/]|*default)"}, } func NewTPCSVFileParser(dirPath, fileName string) (*TPCSVFileParser, error) { diff --git a/engine/loader_helpers_test.go b/engine/loader_helpers_test.go index b8dd0a0a9..57dde2d87 100644 --- a/engine/loader_helpers_test.go +++ b/engine/loader_helpers_test.go @@ -93,17 +93,18 @@ DUMMY,INVALID;DATA cgrates.org,1002;1006,*out,PACKAGE_10,STANDARD_TRIGGERS ` var derivedChargesSample = `#Direction,Tenant,Tor,Account,Subject,RunId,ReqTypeField,DirectionField,TenantField,TorField,AccountField,SubjectField,DestinationField,SetupTimeField,AnswerTimeField,DurationField -*out,cgrates.org,call,dan,dan,extra1,^prepaid,,,,rif,rif,,,, -*out,cgrates.org,,dan,dan,extra1,^prepaid,,,,rif,rif,,,, -*in,cgrates.org,call,dan,dan,extra1,^prepaid,,,,rif,rif,,,, +*out,cgrates.org,call,dan,dan,extra1,^filteredHeader1/filterValue1,^prepaid,,,,rif,rif,,,, +*out,cgrates.org,,dan,dan,extra1,^filteredHeader1/filterValue1,^prepaid,,,,rif,rif,,,, +*in,cgrates.org,call,dan,dan,extra1,^filteredHeader1/filterValue1,^prepaid,,,,rif,rif,,,, DUMMY_DATA -*out,cgrates.org,call,dan,dan,extra2,,,,,ivo,ivo,,,, -*out,cgrates.org,call,dan,*any,extra1,,,,,rif2,rif2,,,, -*out,cgrates.org,call,dan,*any,*any,,,,,rif2,rif2,,,, -*out,cgrates.org,call,dan,*any,*default,*default,*default,*default,*default,rif2,rif2,*default,*default,*default,*default -*out,cgrates.org,call,dan,*any,test,^test,^test,^test,^test,^test,^test,^test,^test,^test,^test -*out,cgrates.org,call,dan,*any,run1,,,,,,,,,, -*out,cgrates.org,call,dan,*default,,,,,,,,,,, +*out,cgrates.org,call,dan,dan,extra2,,,,,,ivo,ivo,,,, +*out,cgrates.org,call,dan,*any,extra1,,,,,,rif2,rif2,,,, +*out,cgrates.org,call,dan,*any,*any,,,,,,rif2,rif2,,,, +*out,cgrates.org,call,dan,*any,*default,,*default,*default,*default,*default,rif2,rif2,*default,*default,*default,*default +*out,cgrates.org,call,dan,*any,test,^test,^test,^test,^test,^test,^test,^test,^test,^test,^test,^test +*out,cgrates.org,call,dan,*any,run1,,,,,,,,,,, +*out,cgrates.org,call,dan,*default,,,,,,,,,,,, +*out,cgrates.org,call,dan,dan,extra3,~filterhdr1:s/(.+)/special_run3/,,,,,^runusr3,^runusr3,,,, ` func TestTimingsValidator(t *testing.T) { @@ -386,7 +387,7 @@ func TestDerivedChargersValidator(t *testing.T) { if valid { t.Error("Validation passed for invalid line", string(ln)) } - case 2, 6, 7, 10, 11: + case 2, 6, 7, 10, 11, 13: if !valid { t.Error("Validation did not pass for valid line", string(ln)) } diff --git a/mediator/mediator.go b/mediator/mediator.go index 6dc813935..d0fccb4f2 100644 --- a/mediator/mediator.go +++ b/mediator/mediator.go @@ -121,8 +121,8 @@ func (self *Mediator) RateCdr(storedCdr *utils.StoredCdr) error { } for _, dc := range dcs { dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) - if dcRunFilter != nil && storedCdr.FieldAsString(&utils.RSRField{Id: dcRunFilter.Id}) != storedCdr.FieldAsString(dcRunFilter) { - engine.Logger.Info(fmt.Sprintf("Ignoring DerivedCharger with id %s due to non matching filter", dc.RunId)) + if !storedCdr.PassesFieldFilter(dcRunFilter) { + engine.Logger.Info(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) continue } dcReqTypeFld, _ := utils.NewRSRField(dc.ReqTypeField) diff --git a/sessionmanager/fssessionmanager.go b/sessionmanager/fssessionmanager.go index 5a5c8685b..161fbfeef 100644 --- a/sessionmanager/fssessionmanager.go +++ b/sessionmanager/fssessionmanager.go @@ -163,7 +163,7 @@ func (sm *FSSessionManager) OnChannelPark(ev Event) { for _, dc := range dcs { dcRunFilter, _ := utils.NewRSRField(dc.RunFilter) if dcRunFilter != nil && ev.ParseEventValue(&utils.RSRField{Id: dcRunFilter.Id}) != ev.ParseEventValue(dcRunFilter) { - engine.Logger.Info(fmt.Sprintf("Ignoring DerivedCharger with id %s due to non matching filter", dc.RunId)) + engine.Logger.Info(fmt.Sprintf(" Ignoring DerivedCharger with id %s - non matching filter", dc.RunId)) } startTime, err := ev.GetAnswerTime(PARK_TIME) if err != nil { diff --git a/utils/consts.go b/utils/consts.go index 6bb86e304..f3dd508d6 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -61,7 +61,7 @@ const ( ACTION_PLANS_NRCOLS = 4 ACTION_TRIGGERS_NRCOLS = 9 ACCOUNT_ACTIONS_NRCOLS = 5 - DERIVED_CHARGERS_NRCOLS = 16 + DERIVED_CHARGERS_NRCOLS = 17 ROUNDING_UP = "*up" ROUNDING_MIDDLE = "*middle" ROUNDING_DOWN = "*down" diff --git a/utils/storedcdr.go b/utils/storedcdr.go index 2c2d03a5a..733b84d80 100644 --- a/utils/storedcdr.go +++ b/utils/storedcdr.go @@ -133,6 +133,17 @@ func (storedCdr *StoredCdr) FieldAsString(rsrFld *RSRField) string { } } +func (storedCdr *StoredCdr) PassesFieldFilter(fieldFilter *RSRField) bool { + if fieldFilter == nil { + return true + } + if storedCdr.FieldAsString(&RSRField{Id: fieldFilter.Id}) == storedCdr.FieldAsString(fieldFilter) && len(storedCdr.FieldAsString(fieldFilter)) != 0 { + // Field value must be non empty in order to declare it filtered, otherwise filter makes no sense + return true + } + return false +} + func (storedCdr *StoredCdr) AsStoredCdr() *StoredCdr { return storedCdr } diff --git a/utils/storedcdr_test.go b/utils/storedcdr_test.go index baf8018a3..6b5c9022a 100644 --- a/utils/storedcdr_test.go +++ b/utils/storedcdr_test.go @@ -78,6 +78,32 @@ func TestFieldAsString(t *testing.T) { } } +func TestPassesFieldFilter(t *testing.T) { + cdr := StoredCdr{CgrId: Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org", + Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: DEFAULT_RUNID, + Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, + } + if !cdr.PassesFieldFilter(nil) { + t.Error("Not passing filter") + } + acntPrefxFltr, _ := NewRSRField(`~account:s/(.+)/1001/`) + if !cdr.PassesFieldFilter(acntPrefxFltr) { + t.Error("Not passing filter") + } + torFltr, _ := NewRSRField(`^tor/*voice`) + if !cdr.PassesFieldFilter(torFltr) { + t.Error("Not passing filter") + } + torFltr, _ = NewRSRField(`^tor/*data`) + if cdr.PassesFieldFilter(torFltr) { + t.Error("Passing filter") + } + inexistentFieldFltr, _ := NewRSRField(`^fakefield/fakevalue`) + if cdr.PassesFieldFilter(inexistentFieldFltr) { + t.Error("Passing filter") + } +} + func TestUsageMultiply(t *testing.T) { cdr := StoredCdr{Usage: time.Duration(10) * time.Second} if cdr.UsageMultiply(1024.0, 0); cdr.Usage != time.Duration(10240)*time.Second {