From c38179c01aa59db5472be49ffe62391e8105a55f Mon Sep 17 00:00:00 2001 From: DanB Date: Sun, 22 Dec 2013 13:06:40 +0100 Subject: [PATCH] Cdrc further refactoring --- cdrc/.cdrc.go.swp | Bin 20480 -> 20480 bytes cdrc/.cdrc_test.go.swp | Bin 12288 -> 0 bytes cdrc/cdrc.go | 43 +++++++++++++++++++++-- cdrc/cdrc_test.go | 3 ++ cdrs/cgrcdr.go | 26 +++++++------- cdrs/cgrcdr_test.go | 2 +- cmd/cgr-engine/cgr-engine.go | 18 ++++++++-- config/config.go | 10 ++++-- config/config_test.go | 5 +-- console/get_cost.go | 7 +--- mediator/mediator.go | 66 ++++------------------------------- test.sh | 7 ++-- utils/cdr.go | 3 +- utils/coreutils.go | 8 +++++ utils/ratedcdr.go | 5 +++ utils/utils_test.go | 25 +++++++++++++ 16 files changed, 135 insertions(+), 93 deletions(-) delete mode 100644 cdrc/.cdrc_test.go.swp diff --git a/cdrc/.cdrc.go.swp b/cdrc/.cdrc.go.swp index 4916112cbc15f0a26f0412cbc505ea65d8e5786a..e9b63cc96965a4e298664cf63432e98482cee5b1 100644 GIT binary patch delta 2079 zcmbu=eTWog9Ki8s)vMd{9ZoSNeeQCP-E-VC1212WqucXkE$?Kn7qZOecAveQ+?zRO zX0vNKRw7NLglj-3C`2q@>Yv;S12eEciZZf-{s{4<7mTF(BQc1;zR%9yX;FW*@Y$K2 zdES1%=eILE4@7nzh%D+~_F#IGY){@L_{4cAx-)&f4`fnn4e6;uKA9bG zQ~7~pD(7bVoT8gm>0F{RXQP~4c9marZE0Ht5girLsBaCaWM4t4;rKs$W{P&VDwrb) zn(3zDSF|>Qa$SAR%1h4Vs*Z5tVb{qj&s%2|R6NgDvd0`!@kC;EwN+EcCOxw((KGM3 za+@J_Bi&oNrahplKfScwZMk71?uI{&ZMv1^=y;PEW#(uS4^G8yi-_R-;>28;cM3V> z%9zKQD@lkRD>xBrF6-aS*w!X{9BCGPXUOzb&XBdWO4H6sPiH`y!(217gJEiFfmYMv zyJmJfgX+M%I2j@C|H_x_=#bU4b{AdKx5|zkUzU(1$pY>fpIts_nbNWy-^!1Oe9_Lz zYWiiX-m-)w_3?yGS#{}jd}vVFGHVtJDo0$d@?>#%JmewAwtUC6>;Xe6%O4~Q6HzkH z@`;HgqYasM(JaXRkuYRB$$sVUPz+zo2bY2;mYiA1me(?nWlhG7lDSbr%%`j1Mh)ne z$)JA8?X$->e0;x0gQ2u6ubbGUoI?!6FV$0)=QkTNXSur27>lZ?m6YQp`+U=n>wAA2 zu%$(EFYWa>{#xBXP6tvf#;b}Bx?IAVdSt8T>+FhpCXf^ zwc_bg6q33rO9si@uB2Dws%2Npi1MuMiWdeZqIzJxmKNq(pDl%xuaa5E&RYYc;okSC z9o)^v!;7*tJSJ1?GpTB`gp5}Uqna4UO$2uuJErIo3l13@qn6Va{x2#&8S^KXV?5V> zfPTUzY7gH-Glh5sFTzJB8t}&q9(VW^XYoAtU>8PF!qZrR>(hm}h}W?jE=+V{Ioj~& zG_hK|$-{XR+mS{G?#72aq-B}~KkJL)>7^uTlg1Lep@dXZI411sheHP)T5}mTU ZE)xAeZ;C5*!JO8W(TVp;<-S%K`3nHd3c&yX delta 2052 zcmbW&Z)_7~9Ki9X8w|TKHZ=^KY&<(tv&OFL>bl|9mc%W@ft$#TY~n(?-nDDiU6*SM z7$G-fV${s&awY^#U?vL6Vj`&_G0}{W_yU+D#1JC^5fa2#hF2trh~H}m*>L>PC;RMq z?&)*CyZha9&oegZ7@Ksg9`wE3)h%1SPYX(=5bqxNdQ+}Qhm=M2KZ5#or}D{DtfI3S%Y06#X=w!&(I4 z!c}hj63*gh?7}t-qZ=!6c99SVup1K?#|YeTK|wWEVg;&jp0rG33LjxdaXUe23?YCj zEXVicdmI7yQG;)8<H}Ov(t$j#M_ON99v?hEZrL9^rH*Vs=MI=O$Sfl6oR3U#FA5$V-V4&9haug=B`7 zcVomzYxRoX=WA*3H8uGBvdIjz`I_5&0mUN?Et)daEom(o^TrZMwM$obXz49VL{D$m zjIDMAWnjQf+{VU?N8U5+zoG{06Ya++T*V_(!}e`Ots}vpbv{(RxF<5AxvlHLDeGpa z)p{*_%-SDnbZXnY`5)X?cM9(?JfroC>+ziK+ULmqxZy>I!)fj9c_w$U zr`BP;`BXu&zBnqA1w3plXkD1sWK!ZqobOx781r#6_4*T+UO-d;XxJu~;-=e8dx8@J!Qcd=H;8_ZQ(D^Zk9SK{J|QFa2+#56_|uOK^f^ zeGEI{$7z=IQ?Qr#qd0Ya$ZH_cXpYRFrZ^2x)DYY?YQy~Cu0xZhKZff&;l1$ z;W2#2-f1Lea& zpz+FYuX4BDwyy0CsUZSHfCvx)B0vO)01+SpM1Tko0U|&IwjhBZV(jH^##(z2JpTXh zz5o9>z}R=_6Z8>!4LyaPK#!q&&;k@d*P!FjE@&sT1A4z7c2EPl3|)jSK@}`i901+SpM1Tko0U|&I{vQG> zD{KIt0qssHm8p_*Jr(zJ5o!_f`ecJI21#3RB|;$=MdY01I39bpn9fwooe{d!)-oyF z#?;lhn^&gFWi}w=brw$1&htbH<&^o63Em(24qyJqORo*iuAVEA2Qu2Y_H-ZdV!%kn6wNw76KRPm2 zs~U@n=U0ck%8*y({@D1bk#Wy6)3u8E?rlfM$7sd%Yp(D5&iXo71rlc1ew{qvk9q1rp z_mWte?Bs+q?H$`vxr0llQkN{1#X#9o<%7mKv%{V@&Ih}$`VP!1_<+-t`c8Yn4YSs8 kw>0~${iQjZ+jN{7qwWqlOt$h&X - CdrcRunDelay int // Sleep interval in seconds between consecutive runs, 0 to use automation via inotify + CdrcRunDelay time.Duration // Sleep interval between consecutive runs, if time unit missing, defaults to seconds, 0 to use automation via inotify CdrcCdrType string // CDR file format . CdrcCdrInDir string // Absolute path towards the directory where the CDRs are stored. CdrcCdrOutDir string // Absolute path towards the directory where processed CDRs will be moved. @@ -169,7 +170,7 @@ func (self *CGRConfig) setDefaults() error { self.CdrcEnabled = false self.CdrcCdrs = "127.0.0.1:2022" self.CdrcCdrsMethod = "http_cgr" - self.CdrcRunDelay = 0 + self.CdrcRunDelay = time.Duration(0) self.CdrcCdrType = "csv" self.CdrcCdrInDir = "/var/log/cgrates/cdr/in/csv" self.CdrcCdrOutDir = "/var/log/cgrates/cdr/out/csv" @@ -372,7 +373,10 @@ func loadConfig(c *conf.ConfigFile) (*CGRConfig, error) { cfg.CdrcCdrsMethod, _ = c.GetString("cdrc", "cdrs_method") } if hasOpt = c.HasOption("cdrc", "run_delay"); hasOpt { - cfg.CdrcRunDelay, _ = c.GetInt("cdrc", "run_delay") + durStr,_ := c.GetString("cdrc", "run_delay") + if cfg.CdrcRunDelay, errParse = utils.ParseDurationWithSecs(durStr); errParse != nil { + return nil, errParse + } } if hasOpt = c.HasOption("cdrc", "cdr_type"); hasOpt { cfg.CdrcCdrType, _ = c.GetString("cdrc", "cdr_type") diff --git a/config/config_test.go b/config/config_test.go index 06e8d3f0b..cc3f7716c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -22,6 +22,7 @@ import ( "fmt" "reflect" "testing" + "time" "github.com/cgrates/cgrates/utils" ) @@ -73,7 +74,7 @@ func TestDefaults(t *testing.T) { eCfg.CdrcEnabled = false eCfg.CdrcCdrs = "127.0.0.1:2022" eCfg.CdrcCdrsMethod = "http_cgr" - eCfg.CdrcRunDelay = 0 + eCfg.CdrcRunDelay = time.Duration(0) eCfg.CdrcCdrType = "csv" eCfg.CdrcCdrInDir = "/var/log/cgrates/cdr/in/csv" eCfg.CdrcCdrOutDir = "/var/log/cgrates/cdr/out/csv" @@ -197,7 +198,7 @@ func TestConfigFromFile(t *testing.T) { eCfg.CdrcEnabled = true eCfg.CdrcCdrs = "test" eCfg.CdrcCdrsMethod = "test" - eCfg.CdrcRunDelay = 99 + eCfg.CdrcRunDelay = time.Duration(99)*time.Second eCfg.CdrcCdrType = "test" eCfg.CdrcCdrInDir = "test" eCfg.CdrcCdrOutDir = "test" diff --git a/console/get_cost.go b/console/get_cost.go index 1d0a691fa..a745b1e9d 100644 --- a/console/get_cost.go +++ b/console/get_cost.go @@ -21,7 +21,6 @@ package console import ( "fmt" "time" - "strconv" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" ) @@ -71,11 +70,7 @@ func (self *CmdGetCost) FromArgs(args []string) error { return fmt.Errorf(self.Usage("")) } } - if _, err := strconv.Atoi(args[7]); err == nil { // No suffix, default to seconds - fmt.Println("\n*duration* needs suffix, defaulting to *s* (valid suffixes: ns, us/µs, ms, s, m, h)\n") - args[7] += "s" - } - callDur, err := time.ParseDuration(args[7]) + callDur, err := utils.ParseDurationWithSecs(args[7]) if err != nil { fmt.Println("\n\tExample durations: 60s for 60 seconds, 25m for 25minutes, 1m25s for one minute and 25 seconds\n") return fmt.Errorf(self.Usage("")) diff --git a/mediator/mediator.go b/mediator/mediator.go index a86ccfd98..3e5755c23 100644 --- a/mediator/mediator.go +++ b/mediator/mediator.go @@ -19,18 +19,12 @@ along with this program. If not, see package mediator import ( - "bufio" - "encoding/csv" "errors" - "flag" "fmt" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" - "os" - "path" "strconv" - "strings" "time" ) @@ -41,11 +35,9 @@ func NewMediator(connector engine.Connector, logDb engine.LogStorage, cdrDb engi cdrDb: cdrDb, cgrCfg: cfg, } - m.fieldNames = make(map[string][]string) - m.fieldIdxs = make(map[string][]int) - // Load config fields - if errLoad := m.loadConfig(); errLoad != nil { - return nil, errLoad + // Parse config + if err := m.parseConfig(); err != nil { + return nil, err } return m, nil } @@ -55,16 +47,10 @@ type Mediator struct { logDb engine.LogStorage cdrDb engine.CdrStorage cgrCfg *config.CGRConfig - cdrInDir, cdrOutDir string - accIdField string - accIdIdx int // Populated only for csv files where we have no names but indexes for the fields - fieldNames map[string][]string - fieldIdxs map[string][]int // Populated only for csv files where we have no names but indexes for the fields } /* -Responsible for loading configuration fields out of CGRConfig instance, doing the necessary pre-checks. -@param fieldKeys: stores the keys which will be referenced inside fieldNames/fieldIdxs +Responsible for parsing configuration fields out of CGRConfig instance, doing the necessary pre-checks. @param cfgVals: keep ordered references to configuration fields from CGRConfig instance. fieldKeys and cfgVals are directly related through index. Method logic: @@ -73,14 +59,12 @@ Method logic: * Accounting id field should not be empty. * If we run mediation on csv file: * Make sure cdrInDir and cdrOutDir are valid paths. - * Populate accIdIdx by converting accIdField into integer. * Populate fieldIdxs by converting fieldNames into integers */ -func (self *Mediator) loadConfig() error { - fieldKeys := []string{"subject", "reqtype", "direction", "tenant", "tor", "account", "destination", "time_start", "duration"} +func (self *Mediator) parseConfig() error { cfgVals := [][]string{self.cgrCfg.MediatorSubjectFields, self.cgrCfg.MediatorReqTypeFields, self.cgrCfg.MediatorDirectionFields, self.cgrCfg.MediatorTenantFields, self.cgrCfg.MediatorTORFields, self.cgrCfg.MediatorAccountFields, self.cgrCfg.MediatorDestFields, - self.cgrCfg.MediatorTimeAnswerFields, self.cgrCfg.MediatorDurationFields} + self.cgrCfg.MediatorAnswerTimeFields, self.cgrCfg.MediatorDurationFields} if len(self.cgrCfg.MediatorRunIds) == 0 { return errors.New("Unconfigured mediator run_ids") @@ -93,44 +77,6 @@ func (self *Mediator) loadConfig() error { } } - // AccIdField has no special requirements, should just exist - if len(self.cgrCfg.MediatorAccIdField) == 0 { - return errors.New("Undefined mediator accid field") - } - self.accIdField = self.cgrCfg.MediatorAccIdField - - var errConv error - // Specific settings of CSV style CDRS - if self.cgrCfg.MediatorCDRType == utils.FSCDR_FILE_CSV { - // Check paths to be valid before adding as configuration - if _, err := os.Stat(self.cgrCfg.MediatorCDRInDir); err != nil { - return fmt.Errorf("The input path for mediator does not exist: %v", self.cgrCfg.MediatorCDRInDir) - } else { - self.cdrInDir = self.cgrCfg.MediatorCDRInDir - } - if _, err := os.Stat(self.cgrCfg.MediatorCDROutDir); err != nil { - return fmt.Errorf("The output path for mediator does not exist: %v", self.cgrCfg.MediatorCDROutDir) - } else { - self.cdrOutDir = self.cgrCfg.MediatorCDROutDir - } - if self.accIdIdx, errConv = strconv.Atoi(self.cgrCfg.MediatorAccIdField); errConv != nil { - return errors.New("AccIdIndex must be integer.") - } - } - - // Load here field names and convert to integers in case of unamed cdrs like CSV - for idx, key := range fieldKeys { - self.fieldNames[key] = cfgVals[idx] - if self.cgrCfg.MediatorCDRType == utils.FSCDR_FILE_CSV { // Special case when field names represent indexes of their location in file - self.fieldIdxs[key] = make([]int, len(cfgVals[idx])) - for iStr, cfgStr := range cfgVals[idx] { - if self.fieldIdxs[key][iStr], errConv = strconv.Atoi(cfgStr); errConv != nil { - return fmt.Errorf("All mediator index members (%s) must be ints", key) - } - } - } - } - return nil } diff --git a/test.sh b/test.sh index 4355e005d..19add0394 100755 --- a/test.sh +++ b/test.sh @@ -8,6 +8,7 @@ go test -i github.com/cgrates/cgrates/mediator go test -i github.com/cgrates/fsock go test -i github.com/cgrates/cgrates/cache2go go test -i github.com/cgrates/cgrates/cdrs +go test -i github.com/cgrates/cgrates/cdrc go test -i github.com/cgrates/cgrates/utils go test -i github.com/cgrates/cgrates/history go test -i github.com/cgrates/cgrates/cdrexporter @@ -23,7 +24,9 @@ cr=$? go test github.com/cgrates/cgrates/mediator md=$? go test github.com/cgrates/cgrates/cdrs -cdr=$? +cdrs=$? +go test github.com/cgrates/cgrates/cdrc +cdrcs=$? go test github.com/cgrates/cgrates/utils ut=$? go test github.com/cgrates/fsock @@ -36,4 +39,4 @@ go test github.com/cgrates/cgrates/cdrexporter cdre=$? -exit $en && $sm && $cfg && $bl && $cr && $md && $cdr && $fs && $ut && $hs && $c2g && $cdre +exit $en && $sm && $cfg && $bl && $cr && $md && $cdrs && $cdrc && $fs && $ut && $hs && $c2g && $cdre diff --git a/utils/cdr.go b/utils/cdr.go index dc1f179aa..1dda1e7b3 100644 --- a/utils/cdr.go +++ b/utils/cdr.go @@ -22,12 +22,13 @@ import ( "time" ) -var PrimaryCdrFields []string = []string{ACCID, CDRHOST, REQTYPE, DIRECTION, TENANT, TOR, ACCOUNT, SUBJECT, DESTINATION, ANSWER_TIME, DURATION} +var PrimaryCdrFields []string = []string{ACCID, CDRHOST, CDRSOURCE, REQTYPE, DIRECTION, TENANT, TOR, ACCOUNT, SUBJECT, DESTINATION, ANSWER_TIME, DURATION} type CDR interface { GetCgrId() string GetAccId() string GetCdrHost() string + GetCdrSource() string GetDirection() string GetSubject() string GetAccount() string diff --git a/utils/coreutils.go b/utils/coreutils.go index d1407ff10..d88de4767 100644 --- a/utils/coreutils.go +++ b/utils/coreutils.go @@ -148,3 +148,11 @@ func CopyHour(src, dest time.Time) time.Time { } return time.Date(dest.Year(), dest.Month(), dest.Day(), src.Hour(), src.Minute(), src.Second(), src.Nanosecond(), src.Location()) } + +// Parses duration, considers s as time unit if not provided +func ParseDurationWithSecs(durStr string) (time.Duration, error) { + if _, err := strconv.Atoi(durStr); err == nil { // No suffix, default to seconds + durStr += "s" + } + return time.ParseDuration(durStr) +} diff --git a/utils/ratedcdr.go b/utils/ratedcdr.go index 94801062f..67f2228f0 100644 --- a/utils/ratedcdr.go +++ b/utils/ratedcdr.go @@ -27,6 +27,7 @@ type RatedCDR struct { CgrId string AccId string CdrHost string + CdrSource string ReqType string Direction string Tenant string @@ -52,6 +53,10 @@ func (ratedCdr *RatedCDR) GetCdrHost() string { return ratedCdr.CdrHost } +func (ratedCdr *RatedCDR) GetCdrSource() string { + return ratedCdr.CdrSource +} + func (ratedCdr *RatedCDR) GetDirection() string { return ratedCdr.Direction } diff --git a/utils/utils_test.go b/utils/utils_test.go index d642bac42..b6ee70cd8 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -247,3 +247,28 @@ func TestSplitPrefixEmpty(t *testing.T) { t.Error("Error splitting prefix: ", a) } } + +func TestParseDurationWithSecs(t *testing.T) { + durStr := "2" + durExpected := time.Duration(2)*time.Second + if parsed, err := ParseDurationWithSecs(durStr); err != nil { + t.Error(err) + } else if parsed != durExpected { + t.Error("Parsed different than expected") + } + durStr = "2s" + if parsed, err := ParseDurationWithSecs(durStr); err != nil { + t.Error(err) + } else if parsed != durExpected { + t.Error("Parsed different than expected") + } + durStr = "2ms" + durExpected = time.Duration(2)*time.Millisecond + if parsed, err := ParseDurationWithSecs(durStr); err != nil { + t.Error(err) + } else if parsed != durExpected { + t.Error("Parsed different than expected") + } +} + +