/* Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments Copyright (C) ITsysCOM GmbH This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see */ package cdrc import ( "encoding/json" "errors" "fmt" "io" "strings" "time" "github.com/antchfx/xmlquery" "github.com/cgrates/cgrates/config" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" ) // handlerUsageDiff will calculate the usage as difference between timeEnd and timeStart // Expects the 2 arguments in template separated by | func handlerSubstractUsage(xmlElement *xmlquery.Node, argsTpl config.RSRParsers, cdrPath utils.HierarchyPath, timezone string) (time.Duration, error) { var argsStr string for _, rsrArg := range argsTpl { if rsrArg.Rules == utils.HandlerArgSep { argsStr += rsrArg.Rules continue } absolutePath := utils.ParseHierarchyPath(rsrArg.Rules, "") relPath := utils.HierarchyPath(absolutePath[len(cdrPath)+1:]) // Need relative path to the xmlElmnt argStr, _ := config.ElementText(xmlElement, relPath.AsString("/", false)) argsStr += argStr } handlerArgs := strings.Split(argsStr, utils.HandlerArgSep) if len(handlerArgs) != 2 { return time.Duration(0), errors.New("Unexpected number of arguments") } tEnd, err := utils.ParseTimeDetectLayout(handlerArgs[0], timezone) if err != nil { return time.Duration(0), err } if tEnd.IsZero() { return time.Duration(0), fmt.Errorf("EndTime is 0") } tStart, err := utils.ParseTimeDetectLayout(handlerArgs[1], timezone) if err != nil { return time.Duration(0), err } if tStart.IsZero() { return time.Duration(0), fmt.Errorf("StartTime is 0") } return tEnd.Sub(tStart), nil } func NewXMLRecordsProcessor(recordsReader io.Reader, cdrPath utils.HierarchyPath, timezone string, httpSkipTlsCheck bool, cdrcCfgs []*config.CdrcCfg, filterS *engine.FilterS) (*XMLRecordsProcessor, error) { //create doc doc, err := xmlquery.Parse(recordsReader) if err != nil { return nil, err } xmlProc := &XMLRecordsProcessor{cdrPath: cdrPath, timezone: timezone, httpSkipTlsCheck: httpSkipTlsCheck, cdrcCfgs: cdrcCfgs, filterS: filterS} xmlProc.cdrXmlElmts = xmlquery.Find(doc, cdrPath.AsString("/", true)) return xmlProc, nil } type XMLRecordsProcessor struct { cdrXmlElmts []*xmlquery.Node // result of splitting the XML doc into CDR elements procItems int // current number of processed records from file cdrPath utils.HierarchyPath // path towards one CDR element timezone string httpSkipTlsCheck bool cdrcCfgs []*config.CdrcCfg // individual configs for the folder CDRC is monitoring filterS *engine.FilterS } func (xmlProc *XMLRecordsProcessor) ProcessedRecordsNr() int64 { return int64(xmlProc.procItems) } func (xmlProc *XMLRecordsProcessor) ProcessNextRecord() (cdrs []*engine.CDR, err error) { if len(xmlProc.cdrXmlElmts) <= xmlProc.procItems { return nil, io.EOF // have processed all items } cdrs = make([]*engine.CDR, 0) cdrXML := xmlProc.cdrXmlElmts[xmlProc.procItems] xmlProc.procItems += 1 xmlProvider := config.NewXmlProvider(cdrXML, xmlProc.cdrPath, utils.MetaReq) for _, cdrcCfg := range xmlProc.cdrcCfgs { tenant, err := cdrcCfg.Tenant.ParseDataProvider(xmlProvider, utils.NestingSep) if err != nil { return nil, err } if tenant == "" { tenant = config.CgrConfig().GeneralCfg().DefaultTenant } if len(cdrcCfg.Filters) != 0 { if pass, err := xmlProc.filterS.Pass(tenant, cdrcCfg.Filters, xmlProvider); err != nil || !pass { continue // Not passes filters, ignore this CDR } } if cdr, err := xmlProc.recordToCDR(cdrXML, cdrcCfg, tenant); err != nil { return nil, fmt.Errorf(" Failed converting to CDR, error: %s", err.Error()) } else { cdrs = append(cdrs, cdr) } if !cdrcCfg.ContinueOnSuccess { break } } return cdrs, nil } func (xmlProc *XMLRecordsProcessor) recordToCDR(xmlEntity *xmlquery.Node, cdrcCfg *config.CdrcCfg, tenant string) (*engine.CDR, error) { cdr := &engine.CDR{OriginHost: "0.0.0.0", Source: cdrcCfg.CdrSourceId, ExtraFields: make(map[string]string), Cost: -1} var lazyHttpFields []*config.FCTemplate var err error fldVals := make(map[string]string) xmlProvider := config.NewXmlProvider(xmlEntity, xmlProc.cdrPath, utils.MetaReq) for _, cdrFldCfg := range cdrcCfg.ContentFields { if len(cdrFldCfg.Filters) != 0 { if pass, err := xmlProc.filterS.Pass(tenant, cdrFldCfg.Filters, xmlProvider); err != nil || !pass { continue // Not passes filters, ignore this CDR } } if cdrFldCfg.Type == utils.META_COMPOSED { out, err := cdrFldCfg.Value.ParseDataProvider(xmlProvider, utils.NestingSep) if err != nil { return nil, err } fldVals[cdrFldCfg.FieldId] += out } else if cdrFldCfg.Type == utils.META_HTTP_POST { lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of cdr to http server } else if cdrFldCfg.Type == utils.META_HANDLER && cdrFldCfg.HandlerId == utils.HandlerSubstractUsage { usage, err := handlerSubstractUsage(xmlEntity, cdrFldCfg.Value, xmlProc.cdrPath, xmlProc.timezone) if err != nil { return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s, err: %s", xmlEntity, cdrFldCfg.Tag, err.Error()) } fldVals[cdrFldCfg.FieldId] += usage.String() } else { return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type) } if err := cdr.ParseFieldValue(cdrFldCfg.FieldId, fldVals[cdrFldCfg.FieldId], xmlProc.timezone); err != nil { return nil, err } } cdr.CGRID = utils.Sha1(cdr.OriginID, cdr.OriginHost) for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields var outValByte []byte var fieldVal, httpAddr string for _, rsrFld := range httpFieldCfg.Value { if parsed, err := rsrFld.ParseValue(utils.EmptyString); err != nil { return nil, fmt.Errorf("Ignoring record: %v - cannot extract http address, err: %s", xmlEntity, err.Error()) } else { httpAddr += parsed } } var jsn []byte jsn, err = json.Marshal(cdr) if err != nil { return nil, err } if outValByte, err = engine.HttpJsonPost(httpAddr, xmlProc.httpSkipTlsCheck, jsn); err != nil && httpFieldCfg.Mandatory { return nil, err } else { fieldVal = string(outValByte) if len(fieldVal) == 0 && httpFieldCfg.Mandatory { return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag) } if err := cdr.ParseFieldValue(httpFieldCfg.FieldId, fieldVal, xmlProc.timezone); err != nil { return nil, err } } } return cdr, nil }