Refactoring validators to match new csv file structure

cmd/cgr-loader/cgr-loader.go

@@ -87,7 +87,7 @@ func main() {
 		loader = engine.NewDbReader(storDb, dataDb, *tpid)
 	} else if *toStorDb { // Import files from a directory into storDb
 		if *tpid == "" {
-			log.Fatal("TPid required, please define it via -tpid command argument.")
+			log.Fatal("TPid required, please define it via *-tpid* command argument.")
 		}
 		csvImporter := engine.TPCSVImporter{ *tpid, storDb, *dataPath, ',', *verbose }
 		if errImport := csvImporter.Run(); errImport != nil {

data/tariffplans/fs_germany_prep1/AccountActions.csv

@@ -0,0 +1,6 @@
+#Tenant,Account,Direction,ActionTimingsTag,ActionTriggersTag
+cgrates.org,1001,*out,PREPAID_10,STANDARD_TRIGGERS
+cgrates.org,1002,*out,PREPAID_10,STANDARD_TRIGGERS
+cgrates.org,1003,*out,PREPAID_10,STANDARD_TRIGGERS
+cgrates.org,1004,*out,PREPAID_10,STANDARD_TRIGGERS
+cgrates.org,1005,*out,PREPAID_10,STANDARD_TRIGGERS

data/tariffplans/fs_germany_prep1/ActionTimings.csv

@@ -0,0 +1,2 @@
+#Tag,ActionsTag,TimingTag,Weight
+TOPUP_10,PREPAID_10,ASAP,10

data/tariffplans/fs_germany_prep1/ActionTriggers.csv

@@ -0,0 +1,4 @@
+#Tag,BalanceTag,Direction,ThresholdType,ThresholdValue,DestinationTag,ActionsTag,Weight
+STANDARD_TRIGGERS,*monetary,*out,*min_balance,2,,LOG_BALANCE,10
+STANDARD_TRIGGERS,*monetary,*out,*max_balance,20,,LOG_BALANCE,10
+STANDARD_TRIGGERS,*monetary,*out,*max_counter,15,FS_USERS,LOG_BALANCE,10

data/tariffplans/fs_germany_prep1/Actions.csv

@@ -0,0 +1,2 @@
+#ActionsTag,Action,BalanceType,Direction,Units,ExpirationDate,DestinationTag,RateType,RateValue,MinutesWeight,Weight
+PREPAID_10,*topup_reset,*monetary,*out,10,*unlimited,*any,,,,10

data/tariffplans/fs_germany_prep1/DestinationRateTimngs.csv

@@ -0,0 +1,6 @@
+#Tag,DestinationRatesTag,TimingTag,Weight
+RETAIL1,DR_RETAIL_PEAK,PEAK,10
+RETAIL1,DR_RETAIL_OFFPEAK,OFFPEAK_MORNING,10
+RETAIL1,DR_RETAIL_OFFPEAK,OFFPEAK_EVENING,10
+RETAIL1,DR_RETAIL_OFFPEAK,WEEKEND,10
+RETAIL1,DR_FREESWITCH_USERS,ALWAYS,10

data/tariffplans/fs_germany_prep1/DestinationRates.csv

@@ -0,0 +1,6 @@
+#Tag,DestinationsTag,RatesTag
+DR_RETAIL_PEAK,GERMANY,LANDLINE_PEAK
+DR_RETAIL_PEAK,GERMANY_MOBILE,MOBILE_PEAK
+DR_RETAIL_OFFPEAK,GERMANY,LANDLINE_OFFPEAK
+DR_RETAIL_OFFPEAK,GERMANY_MOBILE,MOBILE_OFFPEAK
+DR_FREESWITCH_USERS,FS_USERS,RT_FS_USERS

data/tariffplans/fs_germany_prep1/Destinations.csv

@@ -0,0 +1,6 @@
+#Tag,Prefix
+GERMANY,+49
+GERMANY_MOBILE,+4915
+GERMANY_MOBILE,+4916
+GERMANY_MOBILE,+4917
+FS_USERS,10

data/tariffplans/fs_germany_prep1/README.md

@@ -0,0 +1,15 @@
+CGRateS - FSGermanyPrep1
+==========================
+
+Scenario:
+---------
+
+* Create the necessary timings (always, peak, offpeak, asap).
+* Configure 3 different destinations: GERMANY, GERMANY_MOBILE and FS_USERS.
+* Calls to landline and mobile numbers in Germany will be charged time based (structured in peak and offpeak profiles). Calls to landline during peak times are charged using different rate slots: first minute charged as a whole at one rate, next minutes charged per second at another rate.
+* Calls to FreeSWITCH users will be free and time independent.
+* This rating profile will be valid for any rating subject.
+
+* Create 5 prepaid accounts (equivalent of 5 FreeSWITCH default test users - 1001, 1002, 1003, 1004, 1005).
+* Add to each of the accounts a monetary balance of 10 units.
+* For each balance created, attach 3 triggers to control the balance: log on balance=2, log on balance=20, log on 15 mins talked towards FS_USERS destination.

data/tariffplans/fs_germany_prep1/Rates.csv

@@ -0,0 +1,7 @@
+#Tag,ConnectFee,Rate,RatedUnits,RateIncrements,GroupInterval,RoundingMethod,RoundingDecimals,Weight
+LANDLINE_PEAK,0.02,0.02,60,60,0,*up,4,10
+LANDLINE_PEAK,0.02,0.01,1,1,60,*up,4,10
+MOBILE_PEAK,0.02,0.14,60,60,0,*up,4,10
+LANDLINE_OFFPEAK,1,0,60,60,0,*up,4,10
+MOBILE_OFFPEAK,0.02,0.1,60,60,0,*up,4,10
+RT_FS_USERS,0,0,60,60,0,*up,0,10

data/tariffplans/fs_germany_prep1/RatingProfiles.csv

@@ -0,0 +1,2 @@
+#Tenant,TOR,Direction,Subject,ActivationTime,DestinationRateTimingTag,RatesFallbackSubject
+cgrates.org,call,*out,*any,2012-01-01T00:00:00Z,RETAIL1,

data/tariffplans/fs_germany_prep1/Timings.csv

@@ -0,0 +1,7 @@
+#Tag,Years,Months,MonthDays,WeekDays,Time
+ALWAYS,*any,*any,*any,*any,00:00:00
+ASAP,*any,*any,*any,*any,*asap
+OFFPEAK_MORNING,*any,*any,*any,1;2;3;4;5,00:00:00
+PEAK,*any,*any,*any,1;2;3;4;5,08:00:00
+OFFPEAK_EVENING,*any,*any,*any,1;2;3;4;5,20:00:00
+WEEKENDS,*any,*any,*any,6;7,00:00:00

engine/loader_csv_test.go

@@ -72,17 +72,17 @@ EVENING,P2,WORKDAYS_18,10
 EVENING,P2,WEEKENDS,10
 `
 	ratingProfiles = `
-CUSTOMER_1,0,*out,rif:from:tm,danb,PREMIUM,2012-01-01T00:00:00Z
-CUSTOMER_1,0,*out,rif:from:tm,danb,STANDARD,2012-02-28T00:00:00Z
-CUSTOMER_2,0,*out,danb:87.139.12.167,danb,STANDARD,2012-01-01T00:00:00Z
-CUSTOMER_1,0,*out,danb,,PREMIUM,2012-01-01T00:00:00Z
-vdf,0,*out,rif,,EVENING,2012-01-01T00:00:00Z
-vdf,0,*out,rif,,EVENING,2012-02-28T00:00:00Z
-vdf,0,*out,minu,,EVENING,2012-01-01T00:00:00Z
-vdf,0,*out,*any,,EVENING,2012-02-28T00:00:00Z
-vdf,0,*out,one,,STANDARD,2012-02-28T00:00:00Z
-vdf,0,*out,inf,inf,STANDARD,2012-02-28T00:00:00Z
-vdf,0,*out,fall,one|rif,PREMIUM,2012-02-28T00:00:00Z
+CUSTOMER_1,0,*out,rif:from:tm,2012-01-01T00:00:00Z,PREMIUM,danb
+CUSTOMER_1,0,*out,rif:from:tm,2012-02-28T00:00:00Z,STANDARD,danb
+CUSTOMER_2,0,*out,danb:87.139.12.167,2012-01-01T00:00:00Z,STANDARD,danb
+CUSTOMER_1,0,*out,danb,2012-01-01T00:00:00Z,PREMIUM,
+vdf,0,*out,rif,2012-01-01T00:00:00Z,EVENING,
+vdf,0,*out,rif,2012-02-28T00:00:00Z,EVENING,
+vdf,0,*out,minu,2012-01-01T00:00:00Z,EVENING,
+vdf,0,*out,*any,2012-02-28T00:00:00Z,EVENING,
+vdf,0,*out,one,2012-02-28T00:00:00Z,STANDARD,
+vdf,0,*out,inf,2012-02-28T00:00:00Z,STANDARD,inf
+vdf,0,*out,fall,one,2012-02-28T00:00:00Z,PREMIUM|rif
 `
 	actions = `
 MINI,TOPUP,MINUTES,*out,100,1374239002,NAT,*absolute,0,10,10

engine/loader_helpers.go

@@ -209,35 +209,35 @@ type FileLineRegexValidator struct {
 
 var FileValidators = map[string]*FileLineRegexValidator{ 
 			utils.DESTINATIONS_CSV: &FileLineRegexValidator{ utils.DESTINATIONS_NRCOLS,
-				regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+.?\d*){1}$`),
-				"Tag[0-9A-Za-z_],Prefix[0-9]"},
+				regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\+?\d+.?\d*){1}$`),
+				"Tag([0-9A-Za-z_]),Prefix([0-9])"},
 			utils.TIMINGS_CSV: &FileLineRegexValidator{ utils.TIMINGS_NRCOLS,
-				regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\*all\s*,\s*|(?:\d{1,4};?)+\s*,\s*|\s*,\s*){4}(?:\d{2}:\d{2}:\d{2}|\*asap){1}$`),
-				"Tag[0-9A-Za-z_],Years[0-9;]|*all|<empty>,Months[0-9;]|*all|<empty>,MonthDays[0-9;]|*all|<empty>,WeekDays[0-9;]|*all|<empty>,Time[0-9:]|*asap(00:00:00)"},
+				regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\*any\s*,\s*|(?:\d{1,4};?)+\s*,\s*|\s*,\s*){4}(?:\d{2}:\d{2}:\d{2}|\*asap){1}$`),
+				"Tag([0-9A-Za-z_]),Years([0-9;]|*all|<empty>),Months([0-9;]|*all|<empty>),MonthDays([0-9;]|*all|<empty>),WeekDays([0-9;]|*all|<empty>),Time([0-9:]|*asap)"},
 			utils.RATES_CSV: &FileLineRegexValidator{ utils.RATES_NRCOLS,
-				regexp.MustCompile(`(?:\w+\s*,\s*){2}(?:\d+.?\d*,?){4}$`),
-				"Tag[0-9A-Za-z_],ConnectFee[0-9.],Price[0-9.],PricedUnits[0-9.],RateIncrement[0-9.]"},
+				regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+\.?\d*,){5}(?:\*\w+,){1}(?:\d+\.?\d*,?){2}$`),
+				"Tag([0-9A-Za-z_]),ConnectFee([0-9.]),Rate([0-9.]),RatedUnits([0-9.]),RateIncrement([0-9.])"},
 			utils.DESTINATION_RATES_CSV: &FileLineRegexValidator{ utils.DESTINATION_RATES_NRCOLS,
-				regexp.MustCompile(`(?:\w+\s*,\s*){2}(?:\d+.?\d*,?){4}$`),
-				"Tag[0-9A-Za-z_],DestinationsTag[0-9A-Za-z_],RateTag[0-9A-Za-z_]"},
+				regexp.MustCompile(`(?:\w+\s*,?\s*){3}$`),
+				"Tag([0-9A-Za-z_]),DestinationsTag([0-9A-Za-z_]),RateTag([0-9A-Za-z_])"},
 			utils.DESTRATE_TIMINGS_CSV: &FileLineRegexValidator{ utils.DESTRATE_TIMINGS_NRCOLS,
 				regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+.?\d*){1}$`),
-				"Tag[0-9A-Za-z_],DestinationRatesTag[0-9A-Za-z_],TimingProfile[0-9A-Za-z_],Weight[0-9.]"},
+				"Tag([0-9A-Za-z_]),DestinationRatesTag([0-9A-Za-z_]),TimingProfile([0-9A-Za-z_]),Weight([0-9.])"},
 			utils.RATE_PROFILES_CSV: &FileLineRegexValidator{ utils.RATE_PROFILES_NRCOLS,
-				regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\*all\s*,\s*|[\w:\.]+\s*,\s*){1}(?:\w*\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z){1}$`),
-				"Tenant[0-9A-Za-z_],TOR[0-9],Direction OUT|IN,Subject[0-9A-Za-z_:.]|*all,RatesFallbackSubject[0-9A-Za-z_]|<empty>,RatesTimingTag[0-9A-Za-z_],ActivationTime[[0-9T:X]] (2012-01-01T00:00:00Z)"},
+				regexp.MustCompile(`(?:\w+\s*,\s*){2}(?:\*out\s*,\s*){1}(?:\*any\s*,\s*|\w+\s*,\s*){1}(?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z){1}(?:\w*\s*,?\s*){2}$`),
+				"Tenant([0-9A-Za-z_]),TOR([0-9A-Za-z_]),Direction(*out),Subject([0-9A-Za-z_]|*all),RatesFallbackSubject([0-9A-Za-z_]|<empty>),RatesTimingTag([0-9A-Za-z_]),ActivationTime([0-9T:X])"},
 			utils.ACTIONS_CSV: &FileLineRegexValidator{ utils.ACTIONS_NRCOLS,
-				regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\d+\s*,\s*){1}(?:\w+\s*,\s*|\*all\s*,\s*){1}(?:ABSOLUTE\s*,\s*|PERCENT\s*,\s*|\s*,\s*){1}(?:\d*\.?\d*\s*,?\s*){3}$`),
-				"Tag[0-9A-Za-z_],Action[0-9A-Za-z_],BalanceTag[0-9A-Za-z_],Direction OUT|IN,Units[0-9],DestinationTag[0-9A-Za-z_]|*all,PriceType ABSOLUT|PERCENT,PriceValue[0-9.],MinutesWeight[0-9.],Weight[0-9.]"},
+				regexp.MustCompile(`(?:\w+\s*),(?:\*\w+\s*),(?:\*\w+\s*),(?:\*out\s*),(?:\d+\s*),(?:\*\w+\s*|\+\d+[smh]\s*|\d+\s*),(?:\*any|\w+\s*),(?:\*\w+\s*)?,(?:\d+\.?\d*\s*)?,(?:\d+\.?\d*\s*)?,(?:\d+\.?\d*\s*)$`),
+				"Tag([0-9A-Za-z_]),Action([0-9A-Za-z_]),BalanceType([*a-z_]),Direction(*out),Units([0-9]),ExpiryTime(*[a-z_]|+[0-9][smh]|[0-9])DestinationTag([0-9A-Za-z_]|*all),RateType(*[a-z_]),RateValue([0-9.]),MinutesWeight([0-9.]),Weight([0-9.])"},
 			utils.ACTION_TIMINGS_CSV: &FileLineRegexValidator{ utils.ACTION_TIMINGS_NRCOLS,
 				regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+\.?\d*){1}`),
-				"Tag[0-9A-Za-z_],ActionsTag[0-9A-Za-z_],TimingTag[0-9A-Za-z_],Weight[0-9.]"},
+				"Tag([0-9A-Za-z_]),ActionsTag([0-9A-Za-z_]),TimingTag([0-9A-Za-z_]),Weight([0-9.])"},
 			utils.ACTION_TRIGGERS_CSV: &FileLineRegexValidator{ utils.ACTION_TRIGGERS_NRCOLS,
-				regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:MONETARY\s*,\s*|SMS\s*,\s*|MINUTES\s*,\s*|INTERNET\s*,\s*|INTERNET_TIME\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\d+\.?\d*\s*,\s*){1}(?:\w+\s*,\s*|\*all\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\d+\.?\d*){1}$`),
-				"Tag[0-9A-Za-z_],BalanceTag MONETARY|SMS|MINUTES|INTERNET|INTERNET_TIME,Direction OUT|IN,ThresholdValue[0-9.],DestinationTag[0-9A-Za-z_]|*all,ActionsTag[0-9A-Za-z_],Weight[0-9.]"},
+				regexp.MustCompile(`(?:\w+),(?:\*\w+),(?:\*out),(?:\*\w+),(?:\d+\.?\d*),(?:\w+|\*any)?,(?:\w+),(?:\d+\.?\d*)$`),
+				"Tag([0-9A-Za-z_]),BalanceType(*[a-z_]),Direction(*out),ThresholdType(*[a-z_]),ThresholdValue([0-9]+),DestinationTag([0-9A-Za-z_]|*all),ActionsTag([0-9A-Za-z_]),Weight([0-9]+)"},
 			utils.ACCOUNT_ACTIONS_CSV: &FileLineRegexValidator{ utils.ACCOUNT_ACTIONS_NRCOLS,
-				regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:[\w:.]+\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\w+\s*,?\s*){2}$`),
-				"Tenant[0-9A-Za-z_],Account[0-9A-Za-z_:.],Direction OUT|IN,ActionTimingsTag[0-9A-Za-z_],ActionTriggersTag[0-9A-Za-z_]"},
+				regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\*out\s*,\s*){1}(?:\w+\s*,?\s*){2}$`),
+				"Tenant([0-9A-Za-z_]),Account([0-9A-Za-z_.]),Direction(*out),ActionTimingsTag([0-9A-Za-z_]),ActionTriggersTag([0-9A-Za-z_])"},
 		}
 
 
@@ -257,7 +257,7 @@ func NewTPCSVFileParser(dirPath, fileName string) (*TPCSVFileParser, error) {
 	return &TPCSVFileParser{validator, reader}, nil
 }
 
-// Opens the connection to a file and returns the parsed lines one by one when ParseLine() is called
+// Opens the connection to a file and returns the parsed lines one by one when ParseNextLine() is called
 type TPCSVFileParser struct {
 	validator      *FileLineRegexValidator // Row validator
 	reader         *bufio.Reader // Reader to the file we are interested in