diff --git a/loaders/libloader.go b/loaders/libloader.go index bd42ae3ec..2b9f9159b 100644 --- a/loaders/libloader.go +++ b/loaders/libloader.go @@ -123,26 +123,18 @@ func (cP *csvProvider) FieldAsInterface(fldPath []string) (data interface{}, err } err = nil // cancel previous err - if splt := strings.Split(fldPath[0], utils.MatchLessThan); len(splt) != 1 { - var fileName string - // check for *req prefix - if splt[0] != utils.MetaReq { - return nil, fmt.Errorf("invalid prefix for : %s", fldPath) - } - fileName = splt[1] - // find the last > and compute the name of the file - hasGrThan := false + if strings.HasPrefix(fldPath[0], utils.MetaFile+utils.FilterValStart) { + fileName := strings.TrimPrefix(fldPath[0], utils.MetaFile+utils.FilterValStart) + hasSelEnd := false for _, val := range fldPath[1:] { - if grSplt := strings.Split(val, utils.MatchGreaterThan); len(grSplt) == 1 { - fileName = fileName + utils.NestingSep + val - } else { - fileName = fileName + utils.NestingSep + grSplt[0] - hasGrThan = true + if hasSelEnd = strings.HasSuffix(val, utils.FilterValEnd); hasSelEnd { + fileName = fileName + utils.NestingSep + val[:len(val)-1] break } + fileName = fileName + utils.NestingSep + val } - if !hasGrThan { - return nil, fmt.Errorf("filter rule <%s> needs to end in >", fldPath) + if !hasSelEnd { + return nil, fmt.Errorf("filter rule <%s> needs to end in )", fldPath) } if cP.fileName != fileName { cP.cache.Set(fldPath, nil) diff --git a/loaders/libloader_test.go b/loaders/libloader_test.go index 79fbe9b27..55ec11cd0 100644 --- a/loaders/libloader_test.go +++ b/loaders/libloader_test.go @@ -216,7 +216,7 @@ func TestDataUpdateFromCSVMultiFiles(t *testing.T) { {Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), + Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.INFIELD_SEP), Mandatory: true}, {Tag: "Contexts", Path: "Contexts", @@ -225,15 +225,15 @@ func TestDataUpdateFromCSVMultiFiles(t *testing.T) { {Tag: "Path", Path: "Path", Type: utils.META_COMPOSED, - Value: config.NewRSRParsersMustCompile("~*req.5", utils.INFIELD_SEP)}, + Value: config.NewRSRParsersMustCompile("~*file(File1.csv).5", utils.INFIELD_SEP)}, {Tag: "Initial", Path: "Initial", Type: utils.META_COMPOSED, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.INFIELD_SEP)}, + Value: config.NewRSRParsersMustCompile("~*file(File1.csv).6", utils.INFIELD_SEP)}, {Tag: "Substitute", Path: "Substitute", Type: utils.META_COMPOSED, - Value: config.NewRSRParsersMustCompile("~*req.7", utils.INFIELD_SEP)}, + Value: config.NewRSRParsersMustCompile("~*file(File1.csv).7", utils.INFIELD_SEP)}, {Tag: "Append", Path: "Append", Type: utils.MetaString, diff --git a/loaders/loader.go b/loaders/loader.go index fc7a77aa7..af7885d38 100644 --- a/loaders/loader.go +++ b/loaders/loader.go @@ -69,8 +69,12 @@ func NewLoader(dm *engine.DataManager, cfg *config.LoaderSCfg, } for _, cfgFld := range ldrData.Fields { // add all possible files to be opened for _, cfgFldVal := range cfgFld.Value { - if idx := strings.Index(cfgFldVal.Rules, utils.InInFieldSep); idx != -1 { - ldr.rdrs[ldrData.Type][cfgFldVal.Rules[:idx]] = nil + rule := cfgFldVal.Rules + if !strings.HasPrefix(rule, utils.DynamicDataPrefix+utils.MetaFile+utils.FilterValStart) { + continue + } + if idxEnd := strings.Index(rule, utils.FilterValEnd); idxEnd != -1 { + ldr.rdrs[ldrData.Type][rule[7:idxEnd]] = nil } } } diff --git a/loaders/loader_test.go b/loaders/loader_test.go index c23c56593..f7ee249c7 100644 --- a/loaders/loader_test.go +++ b/loaders/loader_test.go @@ -42,49 +42,49 @@ func TestLoaderProcessContentSingleFile(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaAttributes: { - &config.FCTemplate{Tag: "TenantID", + {Tag: "TenantID", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ProfileID", + {Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "Contexts", + {Tag: "Contexts", Path: "Contexts", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "FilterIDs", + {Tag: "FilterIDs", Path: "FilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.3", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ActivationInterval", + {Tag: "ActivationInterval", Path: "ActivationInterval", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.4", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "AttributeFilterIDs", + {Tag: "AttributeFilterIDs", Path: "AttributeFilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.5", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Path", + {Tag: "Path", Path: "Path", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.6", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Type", + {Tag: "Type", Path: "Type", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.7", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Value", + {Tag: "Value", Path: "Value", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.8", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Blocker", + {Tag: "Blocker", Path: "Blocker", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.9", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Weight", + {Tag: "Weight", Path: "Weight", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.10", utils.INFIELD_SEP)}, @@ -148,29 +148,29 @@ func TestLoaderProcessContentMultiFiles(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaAttributes: { - &config.FCTemplate{Tag: "TenantID", + {Tag: "TenantID", Path: "Tenant", Type: utils.MetaString, Value: config.NewRSRParsersMustCompile("cgrates.org", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ProfileID", + {Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, - Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), + Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "Contexts", + {Tag: "Contexts", Path: "Contexts", Type: utils.MetaString, Value: config.NewRSRParsersMustCompile("*any", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Path", + {Tag: "Path", Path: "Path", Type: utils.META_COMPOSED, - Value: config.NewRSRParsersMustCompile("~*req.6", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Value", + Value: config.NewRSRParsersMustCompile("~*file(File1.csv).6", utils.INFIELD_SEP)}, + {Tag: "Value", Path: "Value", Type: utils.META_COMPOSED, - Value: config.NewRSRParsersMustCompile("~*req.7", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Weight", + Value: config.NewRSRParsersMustCompile("~*file(File1.csv).7", utils.INFIELD_SEP)}, + {Tag: "Weight", Path: "Weight", Type: utils.MetaString, Value: config.NewRSRParsersMustCompile("10", utils.INFIELD_SEP)}, @@ -226,49 +226,49 @@ func TestLoaderProcessResource(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaResources: { - &config.FCTemplate{Tag: "Tenant", + {Tag: "Tenant", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ID", + {Tag: "ID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "FilterIDs", + {Tag: "FilterIDs", Path: "FilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ActivationInterval", + {Tag: "ActivationInterval", Path: "ActivationInterval", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.3", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "TTL", + {Tag: "TTL", Path: "UsageTTL", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.4", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Limit", + {Tag: "Limit", Path: "Limit", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.5", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "AllocationMessage", + {Tag: "AllocationMessage", Path: "AllocationMessage", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.6", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Blocker", + {Tag: "Blocker", Path: "Blocker", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.7", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Stored", + {Tag: "Stored", Path: "Stored", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.8", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Weight", + {Tag: "Weight", Path: "Weight", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.9", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Thresholds", + {Tag: "Thresholds", Path: "Thresholds", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.10", utils.INFIELD_SEP)}, @@ -344,29 +344,29 @@ func TestLoaderProcessFilters(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaFilters: { - &config.FCTemplate{Tag: "Tenant", + {Tag: "Tenant", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ID", + {Tag: "ID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "Type", + {Tag: "Type", Path: "Type", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Element", + {Tag: "Element", Path: "Element", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.3", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Values", + {Tag: "Values", Path: "Values", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.4", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ActivationInterval", + {Tag: "ActivationInterval", Path: "ActivationInterval", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.5", utils.INFIELD_SEP)}, @@ -464,49 +464,49 @@ func TestLoaderProcessThresholds(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaThresholds: { - &config.FCTemplate{Tag: "TenantID", + {Tag: "TenantID", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ProfileID", + {Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "FilterIDs", + {Tag: "FilterIDs", Path: "FilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ActivationInterval", + {Tag: "ActivationInterval", Path: "ActivationInterval", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.3", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "MaxHits", + {Tag: "MaxHits", Path: "MaxHits", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.4", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "MinHits", + {Tag: "MinHits", Path: "MinHits", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.5", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "MinSleep", + {Tag: "MinSleep", Path: "MinSleep", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.6", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Blocker", + {Tag: "Blocker", Path: "Blocker", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.7", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Weight", + {Tag: "Weight", Path: "Weight", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.8", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ActionIDs", + {Tag: "ActionIDs", Path: "ActionIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.9", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Async", + {Tag: "Async", Path: "Async", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.10", utils.INFIELD_SEP)}, @@ -562,58 +562,58 @@ func TestLoaderProcessStats(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaStats: { - &config.FCTemplate{Tag: "TenantID", + {Tag: "TenantID", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ProfileID", + {Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "FilterIDs", + {Tag: "FilterIDs", Path: "FilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ActivationInterval", + {Tag: "ActivationInterval", Path: "ActivationInterval", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.3", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "QueueLength", + {Tag: "QueueLength", Path: "QueueLength", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.4", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "TTL", + {Tag: "TTL", Path: "TTL", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.5", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "MinItems", + {Tag: "MinItems", Path: "MinItems", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.6", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "MetricIDs", + {Tag: "MetricIDs", Path: "MetricIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.7", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "MetricFilterIDs", + {Tag: "MetricFilterIDs", Path: "MetricFilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.8", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Blocker", + {Tag: "Blocker", Path: "Blocker", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.9", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Stored", + {Tag: "Stored", Path: "Stored", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.10", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Weight", + {Tag: "Weight", Path: "Weight", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.11", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ThresholdIDs", + {Tag: "ThresholdIDs", Path: "ThresholdIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.12", utils.INFIELD_SEP)}, @@ -684,69 +684,69 @@ func TestLoaderProcessRoutes(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaRoutes: { - &config.FCTemplate{Tag: "TenantID", + {Tag: "TenantID", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ProfileID", + {Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "FilterIDs", + {Tag: "FilterIDs", Path: "FilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ActivationInterval", + {Tag: "ActivationInterval", Path: "ActivationInterval", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.3", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Sorting", + {Tag: "Sorting", Path: "Sorting", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.4", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "SortingParameters", + {Tag: "SortingParameters", Path: "SortingParameters", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.5", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RouteID", + {Tag: "RouteID", Path: "RouteID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.6", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RouteFilterIDs", + {Tag: "RouteFilterIDs", Path: "RouteFilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.7", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RouteAccountIDs", + {Tag: "RouteAccountIDs", Path: "RouteAccountIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.8", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RouteRatingPlanIDs", + {Tag: "RouteRatingPlanIDs", Path: "RouteRatingplanIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.9", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RouteResourceIDs", + {Tag: "RouteResourceIDs", Path: "RouteResourceIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.10", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RouteStatIDs", + {Tag: "RouteStatIDs", Path: "RouteStatIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.11", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RouteWeight", + {Tag: "RouteWeight", Path: "RouteWeight", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.12", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RouteBlocker", + {Tag: "RouteBlocker", Path: "RouteBlocker", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.13", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RouteParameters", + {Tag: "RouteParameters", Path: "RouteParameters", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.14", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Weight", + {Tag: "Weight", Path: "Weight", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.15", utils.INFIELD_SEP)}, @@ -841,33 +841,33 @@ func TestLoaderProcessChargers(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaChargers: { - &config.FCTemplate{Tag: "TenantID", + {Tag: "TenantID", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ProfileID", + {Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "FilterIDs", + {Tag: "FilterIDs", Path: "FilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ActivationInterval", + {Tag: "ActivationInterval", Path: "ActivationInterval", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.3", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RunID", + {Tag: "RunID", Path: "RunID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.4", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "AttributeIDs", + {Tag: "AttributeIDs", Path: "AttributeIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.5", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Weight", + {Tag: "Weight", Path: "Weight", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.6", utils.INFIELD_SEP)}, @@ -919,81 +919,81 @@ func TestLoaderProcessDispatches(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaDispatchers: { - &config.FCTemplate{ + { Tag: "TenantID", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), Mandatory: true, }, - &config.FCTemplate{ + { Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), Mandatory: true, }, - &config.FCTemplate{ + { Tag: "Subsystems", Path: "Subsystems", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP), }, - &config.FCTemplate{ + { Tag: "FilterIDs", Path: "FilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.3", utils.INFIELD_SEP), }, - &config.FCTemplate{ + { Tag: "ActivationInterval", Path: "ActivationInterval", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.4", utils.INFIELD_SEP), }, - &config.FCTemplate{ + { Tag: "Strategy", Path: "Strategy", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.5", utils.INFIELD_SEP), }, - &config.FCTemplate{ + { Tag: "StrategyParameters", Path: "StrategyParameters", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.6", utils.INFIELD_SEP), }, - &config.FCTemplate{ + { Tag: "ConnID", Path: "ConnID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.7", utils.INFIELD_SEP), }, - &config.FCTemplate{ + { Tag: "ConnFilterIDs", Path: "ConnFilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.8", utils.INFIELD_SEP), }, - &config.FCTemplate{ + { Tag: "ConnWeight", Path: "ConnWeight", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.9", utils.INFIELD_SEP), }, - &config.FCTemplate{ + { Tag: "ConnBlocker", Path: "ConnBlocker", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.10", utils.INFIELD_SEP), }, - &config.FCTemplate{ + { Tag: "ConnParameters", Path: "ConnParameters", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.11", utils.INFIELD_SEP), }, - &config.FCTemplate{ + { Tag: "Weight", Path: "Weight", Type: utils.META_COMPOSED, @@ -1071,33 +1071,33 @@ func TestLoaderProcessDispatcheHosts(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaDispatcherHosts: { - &config.FCTemplate{ + { Tag: "Tenant", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), Mandatory: true, }, - &config.FCTemplate{ + { Tag: "ID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), Mandatory: true, }, - &config.FCTemplate{ + { Tag: "Address", Path: "Address", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP), }, - &config.FCTemplate{ + { Tag: "Transport", Path: "Transport", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.3", utils.INFIELD_SEP), }, - &config.FCTemplate{ + { Tag: "TLS", Path: "TLS", Type: utils.META_COMPOSED, @@ -1159,12 +1159,12 @@ func TestLoaderRemoveContentSingleFile(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaAttributes: { - &config.FCTemplate{Tag: "TenantID", + {Tag: "TenantID", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ProfileID", + {Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), @@ -1243,85 +1243,85 @@ func TestLoaderProcessRateProfile(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaRateProfiles: { - &config.FCTemplate{Tag: "TenantID", + {Tag: "TenantID", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ProfileID", + {Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "FilterIDs", + {Tag: "FilterIDs", Path: "FilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ActivationInterval", + {Tag: "ActivationInterval", Path: "ActivationInterval", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.3", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Weight", + {Tag: "Weight", Path: "Weight", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.4", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ConnectFee", + {Tag: "ConnectFee", Path: "ConnectFee", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.5", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RoundingMethod", + {Tag: "RoundingMethod", Path: "RoundingMethod", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.6", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RoundingDecimals", + {Tag: "RoundingDecimals", Path: "RoundingDecimals", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.7", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "MinCost", + {Tag: "MinCost", Path: "MinCost", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.8", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "MaxCost", + {Tag: "MaxCost", Path: "MaxCost", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.9", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "MaxCostStrategy", + {Tag: "MaxCostStrategy", Path: "MaxCostStrategy", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.10", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateID", + {Tag: "RateID", Path: "RateID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.11", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateFilterIDs", + {Tag: "RateFilterIDs", Path: "RateFilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.12", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateActivationStart", + {Tag: "RateActivationStart", Path: "RateActivationStart", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.13", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateWeight", + {Tag: "RateWeight", Path: "RateWeight", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.14", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateBlocker", + {Tag: "RateBlocker", Path: "RateBlocker", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.15", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateIntervalStart", + {Tag: "RateIntervalStart", Path: "RateIntervalStart", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.16", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateValue", + {Tag: "RateValue", Path: "RateValue", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.17", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateUnit", + {Tag: "RateUnit", Path: "RateUnit", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.18", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateIncrement", + {Tag: "RateIncrement", Path: "RateIncrement", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.19", utils.INFIELD_SEP)}, @@ -1420,85 +1420,85 @@ func TestLoaderProcessRateProfileRates(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaRateProfiles: { - &config.FCTemplate{Tag: "TenantID", + {Tag: "TenantID", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ProfileID", + {Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "FilterIDs", + {Tag: "FilterIDs", Path: "FilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ActivationInterval", + {Tag: "ActivationInterval", Path: "ActivationInterval", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.3", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "Weight", + {Tag: "Weight", Path: "Weight", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.4", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "ConnectFee", + {Tag: "ConnectFee", Path: "ConnectFee", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.5", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RoundingMethod", + {Tag: "RoundingMethod", Path: "RoundingMethod", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.6", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RoundingDecimals", + {Tag: "RoundingDecimals", Path: "RoundingDecimals", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.7", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "MinCost", + {Tag: "MinCost", Path: "MinCost", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.8", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "MaxCost", + {Tag: "MaxCost", Path: "MaxCost", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.9", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "MaxCostStrategy", + {Tag: "MaxCostStrategy", Path: "MaxCostStrategy", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.10", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateID", + {Tag: "RateID", Path: "RateID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.11", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateFilterIDs", + {Tag: "RateFilterIDs", Path: "RateFilterIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.12", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateActivationStart", + {Tag: "RateActivationStart", Path: "RateActivationStart", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.13", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateWeight", + {Tag: "RateWeight", Path: "RateWeight", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.14", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateBlocker", + {Tag: "RateBlocker", Path: "RateBlocker", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.15", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateIntervalStart", + {Tag: "RateIntervalStart", Path: "RateIntervalStart", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.16", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateValue", + {Tag: "RateValue", Path: "RateValue", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.17", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateUnit", + {Tag: "RateUnit", Path: "RateUnit", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.18", utils.INFIELD_SEP)}, - &config.FCTemplate{Tag: "RateIncrement", + {Tag: "RateIncrement", Path: "RateIncrement", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.19", utils.INFIELD_SEP)}, @@ -1666,17 +1666,17 @@ func TestLoaderRemoveRateProfileRates(t *testing.T) { } ldr.dataTpls = map[string][]*config.FCTemplate{ utils.MetaRateProfiles: { - &config.FCTemplate{Tag: "TenantID", + {Tag: "TenantID", Path: "Tenant", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.0", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "ProfileID", + {Tag: "ProfileID", Path: "ID", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.1", utils.INFIELD_SEP), Mandatory: true}, - &config.FCTemplate{Tag: "RateIDs", + {Tag: "RateIDs", Path: "RateIDs", Type: utils.META_COMPOSED, Value: config.NewRSRParsersMustCompile("~*req.2", utils.INFIELD_SEP)}, @@ -1970,3 +1970,62 @@ cgrates.org,RP1, t.Errorf("expecting: %+v,\n received: %+v", utils.ToJSON(eRatePrf3), utils.ToJSON(rcv)) } } + +func TestNewLoaderWithMultiFiles(t *testing.T) { + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) + + ldrCfg := config.CgrConfig().LoaderCfg()[0].Clone() + ldrCfg.Data[0].Fields = []*config.FCTemplate{ + {Tag: "TenantID", + Path: "Tenant", + Type: utils.MetaString, + Value: config.NewRSRParsersMustCompile("cgrates.org", utils.INFIELD_SEP), + Mandatory: true}, + {Tag: "ProfileID", + Path: "ID", + Type: utils.META_COMPOSED, + Value: config.NewRSRParsersMustCompile("~*file(File2.csv).1", utils.INFIELD_SEP), + Mandatory: true}, + {Tag: "Contexts", + Path: "Contexts", + Type: utils.MetaString, + Value: config.NewRSRParsersMustCompile("*any", utils.INFIELD_SEP)}, + {Tag: "Path", + Path: "Path", + Type: utils.META_COMPOSED, + Value: config.NewRSRParsersMustCompile("~*file(File1.csv).6", utils.INFIELD_SEP)}, + {Tag: "Value", + Path: "Value", + Type: utils.META_COMPOSED, + Value: config.NewRSRParsersMustCompile("~*file(File1.csv).7", utils.INFIELD_SEP)}, + {Tag: "Weight", + Path: "Weight", + Type: utils.MetaString, + Value: config.NewRSRParsersMustCompile("10", utils.INFIELD_SEP)}, + } + ldr := NewLoader(engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil), ldrCfg, "", make(chan bool), nil, nil, nil) + + openRdrs := make(utils.StringSet) + for _, rdr := range ldr.rdrs { + for fileName := range rdr { + openRdrs.Add(fileName) + } + } + expected := utils.StringSet{ + "Attributes.csv": {}, + "Chargers.csv": {}, + "DispatcherHosts.csv": {}, + "DispatcherProfiles.csv": {}, + "File1.csv": {}, + "File2.csv": {}, + "Filters.csv": {}, + "RateProfiles.csv": {}, + "Resources.csv": {}, + "Routes.csv": {}, + "Stats.csv": {}, + "Thresholds.csv": {}, + } + if !reflect.DeepEqual(expected, openRdrs) { + t.Errorf("Expected %s,received %s", utils.ToJSON(expected), utils.ToJSON(openRdrs)) + } +} diff --git a/packages/debian/changelog b/packages/debian/changelog index 8851623eb..5abcdd6a5 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -78,6 +78,7 @@ cgrates (0.11.0~dev) UNRELEASED; urgency=medium * [RSRParsers] Added grave accent(`) char as a delimiter to not split tge RSR value * [RSRParsers] Moved RSRFilter from RSRParsers to the *rsr FilterS * [SessionS] Rename from ResourceMessage to ResourceAllocation + * [LoaderS] Updated file selector from *req to *file(FileName) -- DanB Wed, 19 Feb 2020 13:25:52 +0200 diff --git a/utils/consts.go b/utils/consts.go index 7c27a961f..aef483284 100755 --- a/utils/consts.go +++ b/utils/consts.go @@ -414,6 +414,7 @@ const ( MetaFileCSV = "*file_csv" MetaVirt = "*virt" MetaFileFWV = "*file_fwv" + MetaFile = "*file" MetaFScsv = "*freeswitch_csv" Accounts = "Accounts" AccountService = "AccountS"