diff --git a/agents/agentreq_test.go b/agents/agentreq_test.go index e9e16251e..02b57e345 100644 --- a/agents/agentreq_test.go +++ b/agents/agentreq_test.go @@ -37,9 +37,9 @@ import ( ) func TestAgReqAsNavigableMap(t *testing.T) { - data := engine.NewInternalDB(nil, nil) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) agReq := NewAgentRequest(nil, nil, nil, nil, nil, "cgrates.org", "", filterS) // populate request, emulating the way will be done in HTTPAgent @@ -133,9 +133,9 @@ func TestAgReqAsNavigableMap(t *testing.T) { } func TestAgReqMaxCost(t *testing.T) { - data := engine.NewInternalDB(nil, nil) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) agReq := NewAgentRequest(nil, nil, nil, nil, nil, "cgrates.org", "", filterS) // populate request, emulating the way will be done in HTTPAgent @@ -177,8 +177,9 @@ func TestAgReqParseFieldDiameter(t *testing.T) { }}) //create diameterDataProvider dP := newDADataProvider(nil, m) - dm := engine.NewDataManager(engine.NewInternalDB(nil, nil), config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + dm := engine.NewDataManager(engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items), + config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) //pass the data provider to agent request agReq := NewAgentRequest(dP, nil, nil, nil, nil, "cgrates.org", "", filterS) @@ -226,9 +227,9 @@ func TestAgReqParseFieldRadius(t *testing.T) { } //create radiusDataProvider dP := newRADataProvider(pkt) - data := engine.NewInternalDB(nil, nil) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) //pass the data provider to agent request agReq := NewAgentRequest(dP, nil, nil, nil, nil, "cgrates.org", "", filterS) @@ -266,9 +267,9 @@ Host: api.cgrates.org } //create radiusDataProvider dP, _ := newHTTPUrlDP(req) - data := engine.NewInternalDB(nil, nil) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) //pass the data provider to agent request agReq := NewAgentRequest(dP, nil, nil, nil, nil, "cgrates.org", "", filterS) @@ -337,8 +338,10 @@ func TestAgReqParseFieldHttpXml(t *testing.T) { } //create radiusDataProvider dP, _ := newHTTPXmlDP(req) - dm := engine.NewDataManager(engine.NewInternalDB(nil, nil), config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + dm := engine.NewDataManager(engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items), + config.CgrConfig().CacheCfg(), nil) + filterS := engine.NewFilterS(cfg, nil, dm) //pass the data provider to agent request agReq := NewAgentRequest(dP, nil, nil, nil, nil, "cgrates.org", "", filterS) @@ -365,9 +368,9 @@ func TestAgReqParseFieldHttpXml(t *testing.T) { } func TestAgReqEmptyFilter(t *testing.T) { - data := engine.NewInternalDB(nil, nil) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) agReq := NewAgentRequest(nil, nil, nil, nil, nil, "cgrates.org", "", filterS) // populate request, emulating the way will be done in HTTPAgent @@ -408,8 +411,9 @@ func TestAgReqEmptyFilter(t *testing.T) { } func TestAgReqMetaExponent(t *testing.T) { - dm := engine.NewDataManager(engine.NewInternalDB(nil, nil), config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + dm := engine.NewDataManager(engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items), + config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) agReq := NewAgentRequest(nil, nil, nil, nil, nil, "cgrates.org", "", filterS) agReq.CGRRequest.Set([]string{"Value"}, "2", false, false) @@ -433,9 +437,9 @@ func TestAgReqMetaExponent(t *testing.T) { } func TestAgReqCGRActiveRequest(t *testing.T) { - data := engine.NewInternalDB(nil, nil) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) agReq := NewAgentRequest(nil, nil, nil, nil, nil, "cgrates.org", "", filterS) // populate request, emulating the way will be done in HTTPAgent @@ -476,9 +480,9 @@ func TestAgReqCGRActiveRequest(t *testing.T) { } func TestAgReqFieldAsNone(t *testing.T) { - data := engine.NewInternalDB(nil, nil) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) agReq := NewAgentRequest(nil, nil, nil, nil, nil, "cgrates.org", "", filterS) // populate request, emulating the way will be done in HTTPAgent @@ -513,8 +517,9 @@ func TestAgReqFieldAsNone(t *testing.T) { } func TestAgReqFieldAsNone2(t *testing.T) { - dm := engine.NewDataManager(engine.NewInternalDB(nil, nil), config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + dm := engine.NewDataManager(engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items), + config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) agReq := NewAgentRequest(nil, nil, nil, nil, nil, "cgrates.org", "", filterS) // populate request, emulating the way will be done in HTTPAgent @@ -552,9 +557,9 @@ func TestAgReqFieldAsNone2(t *testing.T) { } func TestAgReqAsNavigableMap2(t *testing.T) { - data := engine.NewInternalDB(nil, nil) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) agReq := NewAgentRequest(nil, nil, nil, nil, nil, "cgrates.org", "", filterS) // populate request, emulating the way will be done in HTTPAgent @@ -609,8 +614,9 @@ func TestAgReqAsNavigableMap2(t *testing.T) { } func TestAgReqFieldAsInterface(t *testing.T) { - dm := engine.NewDataManager(engine.NewInternalDB(nil, nil), config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + dm := engine.NewDataManager(engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items), + config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) agReq := NewAgentRequest(nil, nil, nil, nil, nil, "cgrates.org", "", filterS) // populate request, emulating the way will be done in HTTPAgent @@ -655,9 +661,9 @@ func TestAgReqFieldAsInterface(t *testing.T) { } func TestAgReqNewARWithCGRRplyAndRply(t *testing.T) { - data := engine.NewInternalDB(nil, nil) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) ev := map[string]interface{}{ @@ -705,8 +711,9 @@ func TestAgReqNewARWithCGRRplyAndRply(t *testing.T) { } func TestAgReqSetCGRReplyWithError(t *testing.T) { - dm := engine.NewDataManager(engine.NewInternalDB(nil, nil), config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + dm := engine.NewDataManager(engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items), + config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) ev := map[string]interface{}{ @@ -745,9 +752,9 @@ func (ev myEv) AsNavigableMap(tpl []*config.FCTemplate) (*config.NavigableMap, e } func TestAgReqSetCGRReplyWithoutError(t *testing.T) { - data := engine.NewInternalDB(nil, nil) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) ev := map[string]interface{}{ @@ -807,8 +814,9 @@ func TestAgReqParseFieldMetaCCUsage(t *testing.T) { }}) //create diameterDataProvider dP := newDADataProvider(nil, m) - dm := engine.NewDataManager(engine.NewInternalDB(nil, nil), config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + dm := engine.NewDataManager(engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items), + config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) //pass the data provider to agent request agReq := NewAgentRequest(dP, nil, nil, nil, nil, "cgrates.org", "", filterS) @@ -884,8 +892,9 @@ func TestAgReqParseFieldMetaUsageDifference(t *testing.T) { }}) //create diameterDataProvider dP := newDADataProvider(nil, m) - dm := engine.NewDataManager(engine.NewInternalDB(nil, nil), config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + dm := engine.NewDataManager(engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items), + config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) //pass the data provider to agent request agReq := NewAgentRequest(dP, nil, nil, nil, nil, "cgrates.org", "", filterS) @@ -949,9 +958,9 @@ func TestAgReqParseFieldMetaSum(t *testing.T) { }}) //create diameterDataProvider dP := newDADataProvider(nil, m) - data := engine.NewInternalDB(nil, nil) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) //pass the data provider to agent request agReq := NewAgentRequest(dP, nil, nil, nil, nil, "cgrates.org", "", filterS) @@ -993,8 +1002,9 @@ func TestAgReqParseFieldMetaDifference(t *testing.T) { }}) //create diameterDataProvider dP := newDADataProvider(nil, m) - dm := engine.NewDataManager(engine.NewInternalDB(nil, nil), config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + dm := engine.NewDataManager(engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items), + config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) //pass the data provider to agent request agReq := NewAgentRequest(dP, nil, nil, nil, nil, "cgrates.org", "", filterS) @@ -1036,9 +1046,9 @@ func TestAgReqParseFieldMetaValueExponent(t *testing.T) { }}) //create diameterDataProvider dP := newDADataProvider(nil, m) - data := engine.NewInternalDB(nil, nil) - dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dm := engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := engine.NewFilterS(cfg, nil, dm) //pass the data provider to agent request agReq := NewAgentRequest(dP, nil, nil, nil, nil, "cgrates.org", "", filterS) diff --git a/apier/v1/accounts_test.go b/apier/v1/accounts_test.go index 9ada2aaaf..dfc679688 100644 --- a/apier/v1/accounts_test.go +++ b/apier/v1/accounts_test.go @@ -32,8 +32,8 @@ var ( ) func init() { - apierAcntsAcntStorage = engine.NewInternalDB(nil, nil) cfg, _ := config.NewDefaultCGRConfig() + apierAcntsAcntStorage = engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) apierAcnts = &ApierV1{ DataManager: engine.NewDataManager(apierAcntsAcntStorage, config.CgrConfig().CacheCfg(), nil), Config: cfg, diff --git a/apier/v1/debit_test.go b/apier/v1/debit_test.go index 4068a7e68..166b5bbcc 100644 --- a/apier/v1/debit_test.go +++ b/apier/v1/debit_test.go @@ -35,9 +35,10 @@ var ( ) func init() { - apierDebitStorage = engine.NewInternalDB(nil, nil) cfg, _ := config.NewDefaultCGRConfig() config.SetCgrConfig(cfg) + apierDebitStorage = engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + responder := &engine.Responder{MaxComputedUsage: cfg.RalsCfg().MaxComputedUsage} dm = engine.NewDataManager(apierDebitStorage, config.CgrConfig().CacheCfg(), nil) engine.SetDataStorage(dm) @@ -49,9 +50,9 @@ func init() { } func TestDebitUsageWithOptionsSetConfig(t *testing.T) { - apierDebitStorage = engine.NewInternalDB(nil, nil) cfg, _ := config.NewDefaultCGRConfig() config.SetCgrConfig(cfg) + apierDebitStorage = engine.NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) responder := &engine.Responder{MaxComputedUsage: cfg.RalsCfg().MaxComputedUsage} dm = engine.NewDataManager(apierDebitStorage, config.CgrConfig().CacheCfg(), nil) engine.SetDataStorage(dm) diff --git a/cdrc/csv_test.go b/cdrc/csv_test.go index 7653e2d2d..eaa8b174f 100644 --- a/cdrc/csv_test.go +++ b/cdrc/csv_test.go @@ -74,7 +74,7 @@ func TestCsvRecordToCDR(t *testing.T) { func TestCsvDataMultiplyFactor(t *testing.T) { cgrConfig, _ := config.NewDefaultCGRConfig() cdrcConfig := cgrConfig.CdrcProfiles["/var/spool/cgrates/cdrc/in"][0] - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, cgrConfig.DataDbCfg().Items) dm := engine.NewDataManager(data, cgrConfig.CacheCfg(), nil) filterS := engine.NewFilterS(cgrConfig, nil, dm) cdrcConfig.CdrSourceId = "TEST_CDRC" @@ -159,7 +159,7 @@ func TestCsvPairToRecord(t *testing.T) { func TestCsvSecondUsage(t *testing.T) { cgrConfig, _ := config.NewDefaultCGRConfig() cdrcConfig := cgrConfig.CdrcProfiles["/var/spool/cgrates/cdrc/in"][0] - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, cgrConfig.DataDbCfg().Items) dm := engine.NewDataManager(data, cgrConfig.CacheCfg(), nil) filterS := engine.NewFilterS(cgrConfig, nil, dm) cdrcConfig.CdrSourceId = "TEST_CDRC" diff --git a/cdrc/xml_test.go b/cdrc/xml_test.go index 920faee6a..da0739094 100644 --- a/cdrc/xml_test.go +++ b/cdrc/xml_test.go @@ -326,11 +326,8 @@ func TestXMLRPProcessWithNewFilters(t *testing.T) { }, }, } - data := engine.NewInternalDB(nil, nil) - defaultCfg, err := config.NewDefaultCGRConfig() - if err != nil { - t.Errorf("Error: %+v", err) - } + defaultCfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) xmlRP, err := NewXMLRecordsProcessor(bytes.NewBufferString(cdrXmlBroadsoft), utils.HierarchyPath([]string{"broadWorksCDR", "cdrData"}), "UTC", true, cdrcCfgs, engine.NewFilterS(defaultCfg, nil, @@ -572,11 +569,8 @@ func TestXMLRPNestingSeparator(t *testing.T) { }, }, } - data := engine.NewInternalDB(nil, nil) - defaultCfg, err := config.NewDefaultCGRConfig() - if err != nil { - t.Errorf("Error: %+v", err) - } + defaultCfg, _ := config.NewDefaultCGRConfig() + data := engine.NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) xmlRP, err := NewXMLRecordsProcessor(bytes.NewBufferString(xmlContent), utils.HierarchyPath([]string{"File", "CDRs", "Call"}), "UTC", true, cdrcCfgs, engine.NewFilterS(defaultCfg, nil, diff --git a/cmd/cgr-loader/cgr-loader.go b/cmd/cgr-loader/cgr-loader.go index 5a2ee5c16..719ab0812 100755 --- a/cmd/cgr-loader/cgr-loader.go +++ b/cmd/cgr-loader/cgr-loader.go @@ -252,7 +252,7 @@ func main() { ldrCfg.DataDbCfg().DataDbHost, ldrCfg.DataDbCfg().DataDbPort, ldrCfg.DataDbCfg().DataDbName, ldrCfg.DataDbCfg().DataDbUser, ldrCfg.DataDbCfg().DataDbPass, ldrCfg.GeneralCfg().DBDataEncoding, - ldrCfg.DataDbCfg().DataDbSentinelName) + ldrCfg.DataDbCfg().DataDbSentinelName, ldrCfg.DataDbCfg().Items) if err != nil { log.Fatalf("Coud not open dataDB connection: %s", err.Error()) } @@ -267,7 +267,7 @@ func main() { ldrCfg.StorDbCfg().Password, ldrCfg.StorDbCfg().SSLMode, ldrCfg.StorDbCfg().MaxOpenConns, ldrCfg.StorDbCfg().MaxIdleConns, ldrCfg.StorDbCfg().ConnMaxLifetime, ldrCfg.StorDbCfg().StringIndexedFields, - ldrCfg.StorDbCfg().PrefixIndexedFields); err != nil { + ldrCfg.StorDbCfg().PrefixIndexedFields, ldrCfg.StorDbCfg().Items); err != nil { log.Fatalf("Coud not open storDB connection: %s", err.Error()) } defer storDb.Close() diff --git a/cmd/cgr-migrator/cgr-migrator.go b/cmd/cgr-migrator/cgr-migrator.go index ebde73832..fc8da64ad 100755 --- a/cmd/cgr-migrator/cgr-migrator.go +++ b/cmd/cgr-migrator/cgr-migrator.go @@ -230,7 +230,7 @@ func main() { mgrCfg.DataDbCfg().DataDbHost, mgrCfg.DataDbCfg().DataDbPort, mgrCfg.DataDbCfg().DataDbName, mgrCfg.DataDbCfg().DataDbUser, mgrCfg.DataDbCfg().DataDbPass, mgrCfg.GeneralCfg().DBDataEncoding, - mgrCfg.CacheCfg(), mgrCfg.DataDbCfg().DataDbSentinelName); err != nil { + mgrCfg.CacheCfg(), mgrCfg.DataDbCfg().DataDbSentinelName, mgrCfg.DataDbCfg().Items); err != nil { log.Fatal(err) } @@ -240,7 +240,7 @@ func main() { mgrCfg.MigratorCgrCfg().OutDataDBHost, mgrCfg.MigratorCgrCfg().OutDataDBPort, mgrCfg.MigratorCgrCfg().OutDataDBName, mgrCfg.MigratorCgrCfg().OutDataDBUser, mgrCfg.MigratorCgrCfg().OutDataDBPassword, mgrCfg.MigratorCgrCfg().OutDataDBEncoding, - mgrCfg.CacheCfg(), mgrCfg.MigratorCgrCfg().OutDataDBRedisSentinel); err != nil { + mgrCfg.CacheCfg(), mgrCfg.MigratorCgrCfg().OutDataDBRedisSentinel, mgrCfg.DataDbCfg().Items); err != nil { log.Fatal(err) } @@ -319,7 +319,7 @@ func main() { mgrCfg.StorDbCfg().Password, mgrCfg.StorDbCfg().SSLMode, mgrCfg.StorDbCfg().MaxOpenConns, mgrCfg.StorDbCfg().MaxIdleConns, mgrCfg.StorDbCfg().ConnMaxLifetime, mgrCfg.StorDbCfg().StringIndexedFields, - mgrCfg.StorDbCfg().PrefixIndexedFields); err != nil { + mgrCfg.StorDbCfg().PrefixIndexedFields, mgrCfg.StorDbCfg().Items); err != nil { log.Fatal(err) } @@ -331,7 +331,7 @@ func main() { mgrCfg.MigratorCgrCfg().OutStorDBPassword, mgrCfg.StorDbCfg().SSLMode, mgrCfg.StorDbCfg().MaxOpenConns, mgrCfg.StorDbCfg().MaxIdleConns, mgrCfg.StorDbCfg().ConnMaxLifetime, mgrCfg.StorDbCfg().StringIndexedFields, - mgrCfg.StorDbCfg().PrefixIndexedFields); err != nil { + mgrCfg.StorDbCfg().PrefixIndexedFields, mgrCfg.StorDbCfg().Items); err != nil { log.Fatal(err) } diff --git a/cmd/cgr-tester/cgr-tester.go b/cmd/cgr-tester/cgr-tester.go index 1e393a8b6..c9c7c42a8 100644 --- a/cmd/cgr-tester/cgr-tester.go +++ b/cmd/cgr-tester/cgr-tester.go @@ -76,7 +76,7 @@ func durInternalRater(cd *engine.CallDescriptorWithArgDispatcher) (time.Duration tstCfg.DataDbCfg().DataDbHost, tstCfg.DataDbCfg().DataDbPort, tstCfg.DataDbCfg().DataDbName, tstCfg.DataDbCfg().DataDbUser, tstCfg.DataDbCfg().DataDbPass, tstCfg.GeneralCfg().DBDataEncoding, - tstCfg.DataDbCfg().DataDbSentinelName) + tstCfg.DataDbCfg().DataDbSentinelName, tstCfg.DataDbCfg().Items) if err != nil { return nilDuration, fmt.Errorf("Could not connect to data database: %s", err.Error()) } diff --git a/config/config.go b/config/config.go index 44ea79cfe..662cf3102 100755 --- a/config/config.go +++ b/config/config.go @@ -141,7 +141,9 @@ func NewDefaultCGRConfig() (cfg *CGRConfig, err error) { cfg.generalCfg = new(GeneralCfg) cfg.generalCfg.NodeID = utils.UUIDSha1Prefix() cfg.dataDbCfg = new(DataDbCfg) + cfg.dataDbCfg.Items = make(map[string]*ItemOpt) cfg.storDbCfg = new(StorDbCfg) + cfg.storDbCfg.Items = make(map[string]*ItemOpt) cfg.tlsCfg = new(TlsCfg) cfg.cacheCfg = make(CacheCfg) cfg.listenCfg = new(ListenCfg) diff --git a/config/datadbcfg.go b/config/datadbcfg.go index 2e36fb087..824c23b7b 100644 --- a/config/datadbcfg.go +++ b/config/datadbcfg.go @@ -100,7 +100,6 @@ func (dbcfg *DataDbCfg) loadFromJsonCfg(jsnDbCfg *DbJsonCfg) (err error) { } } if jsnDbCfg.Items != nil { - dbcfg.Items = make(map[string]*ItemOpt) for kJsn, vJsn := range *jsnDbCfg.Items { val := new(ItemOpt) if err := val.loadFromJsonCfg(vJsn); err != nil { diff --git a/config/stordbcfg.go b/config/stordbcfg.go index 6f1a780b0..50a44e543 100644 --- a/config/stordbcfg.go +++ b/config/stordbcfg.go @@ -95,7 +95,6 @@ func (dbcfg *StorDbCfg) loadFromJsonCfg(jsnDbCfg *DbJsonCfg) (err error) { dbcfg.SSLMode = *jsnDbCfg.Sslmode } if jsnDbCfg.Items != nil { - dbcfg.Items = make(map[string]*ItemOpt) for kJsn, vJsn := range *jsnDbCfg.Items { val := new(ItemOpt) if err := val.loadFromJsonCfg(vJsn); err != nil { diff --git a/data/conf/samples/internal_ttl/cgrates.json b/data/conf/samples/internal_ttl/cgrates.json new file mode 100644 index 000000000..578ba71a7 --- /dev/null +++ b/data/conf/samples/internal_ttl/cgrates.json @@ -0,0 +1,55 @@ +{ + + +"general": { + "log_level": 7, +}, + + +"listen": { + "rpc_json": ":2012", + "rpc_gob": ":2013", + "http": ":2080", +}, + + +"data_db": { + "db_type": "*internal", +}, + + +"stor_db": { + "db_type": "*internal", + "items":{ + "cdrs": {"ttl": "2s"} + } +}, + + +"rals": { + "enabled": true, +}, + + +"schedulers": { + "enabled": true, + "cdrs_conns": ["*internal"], +}, + + +"cdrs": { + "enabled": true, +}, + + +"chargers": { + "enabled": true, +}, + + +"apier": { + "scheduler_conns": ["*internal"], +}, + + +} diff --git a/engine/attributes_test.go b/engine/attributes_test.go index b837d5d68..d7d88ca43 100644 --- a/engine/attributes_test.go +++ b/engine/attributes_test.go @@ -148,15 +148,13 @@ var ( ) func TestAttributePopulateAttrService(t *testing.T) { - data := NewInternalDB(nil, nil) - dmAtr = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - defaultCfg, err := config.NewDefaultCGRConfig() + defaultCfg, _ := config.NewDefaultCGRConfig() defaultCfg.AttributeSCfg().ProcessRuns = 1 defaultCfg.AttributeSCfg().StringIndexedFields = nil defaultCfg.AttributeSCfg().PrefixIndexedFields = nil - if err != nil { - t.Errorf("Error: %+v", err) - } + defaultCfg.AttributeSCfg().PrefixIndexedFields = nil + data := NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) + dmAtr = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) attrService, err = NewAttributeService(dmAtr, &FilterS{dm: dmAtr, cfg: defaultCfg}, defaultCfg) if err != nil { t.Errorf("Error: %+v", err) @@ -1695,13 +1693,10 @@ func TestAttributeProcessEventValueExponent(t *testing.T) { } func BenchmarkAttributeProcessEventConstant(b *testing.B) { - data := NewInternalDB(nil, nil) - dmAtr = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - defaultCfg, err := config.NewDefaultCGRConfig() + defaultCfg, _ := config.NewDefaultCGRConfig() defaultCfg.AttributeSCfg().ProcessRuns = 1 - if err != nil { - b.Errorf("Error: %+v", err) - } + data := NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) + dmAtr = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) attrService, err = NewAttributeService(dmAtr, &FilterS{dm: dmAtr, cfg: defaultCfg}, defaultCfg) if err != nil { b.Errorf("Error: %+v", err) @@ -1759,13 +1754,10 @@ func BenchmarkAttributeProcessEventConstant(b *testing.B) { } func BenchmarkAttributeProcessEventVariable(b *testing.B) { - data := NewInternalDB(nil, nil) - dmAtr = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - defaultCfg, err := config.NewDefaultCGRConfig() + defaultCfg, _ := config.NewDefaultCGRConfig() defaultCfg.AttributeSCfg().ProcessRuns = 1 - if err != nil { - b.Errorf("Error: %+v", err) - } + data := NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) + dmAtr = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) attrService, err = NewAttributeService(dmAtr, &FilterS{dm: dmAtr, cfg: defaultCfg}, defaultCfg) if err != nil { b.Errorf("Error: %+v", err) diff --git a/engine/calldesc.go b/engine/calldesc.go index 264f4f408..473e1101a 100644 --- a/engine/calldesc.go +++ b/engine/calldesc.go @@ -46,7 +46,7 @@ func init() { cgrCfg, _ = config.NewDefaultCGRConfig() config.SetCgrConfig(cgrCfg) } - data = NewInternalDB(nil, nil) + data = NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) } dm = NewDataManager(data, config.CgrConfig().CacheCfg(), connMgr) } diff --git a/engine/cdr_test.go b/engine/cdr_test.go index b858be968..c1457b57a 100644 --- a/engine/cdr_test.go +++ b/engine/cdr_test.go @@ -705,13 +705,9 @@ func TestCDRAsExportRecord(t *testing.T) { } else if expRecord[0] != "1" { t.Errorf("Expecting:\n%s\nReceived:\n%s", "1", expRecord[0]) } - data := NewInternalDB(nil, nil) + defaultCfg, _ := config.NewDefaultCGRConfig() + data := NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) dmForCDR := NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - defaultCfg, err := config.NewDefaultCGRConfig() - if err != nil { - t.Errorf("Error: %+v", err) - } - cfgCdrFld = &config.FCTemplate{Tag: "destination", Type: utils.META_COMPOSED, FieldId: utils.Destination, Value: prsr, Filters: []string{"*string:~*req.Tenant:itsyscom.com"}, Timezone: "UTC"} if rcrd, err := cdr.AsExportRecord([]*config.FCTemplate{cfgCdrFld}, false, nil, &FilterS{dm: dmForCDR, cfg: defaultCfg}); err != nil { diff --git a/engine/chargers_test.go b/engine/chargers_test.go index de1b312af..2056a327b 100755 --- a/engine/chargers_test.go +++ b/engine/chargers_test.go @@ -101,12 +101,9 @@ var ( ) func TestChargerPopulateChargerService(t *testing.T) { - data := NewInternalDB(nil, nil) + defaultCfg, _ := config.NewDefaultCGRConfig() + data := NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) dmCharger = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - defaultCfg, err := config.NewDefaultCGRConfig() - if err != nil { - t.Errorf("Error: %+v", err) - } chargerSrv, err = NewChargerService(dmCharger, &FilterS{dm: dmCharger, cfg: defaultCfg}, defaultCfg, nil) if err != nil { diff --git a/engine/datamanager.go b/engine/datamanager.go index 5c15bc88e..fe8860756 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -2244,7 +2244,7 @@ func (dm *DataManager) SetLoadIDs(loadIDs map[string]int64) (err error) { // Reconnect reconnects to the DB when the config was changed func (dm *DataManager) Reconnect(marshaller string, newcfg *config.DataDbCfg) (err error) { d, err := NewDataDBConn(newcfg.DataDbType, newcfg.DataDbHost, newcfg.DataDbPort, newcfg.DataDbName, - newcfg.DataDbUser, newcfg.DataDbPass, marshaller, newcfg.DataDbSentinelName) + newcfg.DataDbUser, newcfg.DataDbPass, marshaller, newcfg.DataDbSentinelName, newcfg.Items) if err != nil { return } diff --git a/engine/filterhelpers_test.go b/engine/filterhelpers_test.go index 324c30500..169de246c 100644 --- a/engine/filterhelpers_test.go +++ b/engine/filterhelpers_test.go @@ -36,7 +36,7 @@ func TestFilterMatchingItemIDsForEvent(t *testing.T) { var stringFilter, prefixFilter, defaultFilter []*FilterRule stringFilterID := "stringFilterID" prefixFilterID := "prefixFilterID" - data := NewInternalDB(nil, nil) + data := NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) dmMatch = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) context := utils.MetaRating x, err := NewFilterRule(utils.MetaString, "~*req.Field", []string{"profile"}) @@ -108,7 +108,7 @@ func TestFilterMatchingItemIDsForEvent2(t *testing.T) { var stringFilter, prefixFilter, defaultFilter []*FilterRule stringFilterID := "stringFilterID" prefixFilterID := "prefixFilterID" - data := NewInternalDB(nil, nil) + data := NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) dmMatch = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) context := utils.MetaRating x, err := NewFilterRule(utils.MetaString, "~*req.CallCost.Account", []string{"1001"}) diff --git a/engine/filterindexer_it_test.go b/engine/filterindexer_it_test.go index 618017291..3217e4957 100644 --- a/engine/filterindexer_it_test.go +++ b/engine/filterindexer_it_test.go @@ -100,7 +100,8 @@ func TestFilterIndexerITMongo(t *testing.T) { } func TestFilterIndexerITInternal(t *testing.T) { - dataManager = NewDataManager(NewInternalDB(nil, nil), config.CgrConfig().CacheCfg(), nil) + dataManager = NewDataManager(NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items), + config.CgrConfig().CacheCfg(), nil) for _, stest := range sTests { t.Run("TestITInternal", stest) } diff --git a/engine/filters_test.go b/engine/filters_test.go index 91daaf42f..f8f1c0975 100644 --- a/engine/filters_test.go +++ b/engine/filters_test.go @@ -604,9 +604,9 @@ func TestFilterNewRequestFilter(t *testing.T) { } func TestInlineFilterPassFiltersForEvent(t *testing.T) { - data := NewInternalDB(nil, nil) - dmFilterPass := NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dmFilterPass := NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := FilterS{ cfg: cfg, dm: dmFilterPass, @@ -816,9 +816,9 @@ func TestInlineFilterPassFiltersForEvent(t *testing.T) { } func TestPassFiltersForEventWithEmptyFilter(t *testing.T) { - data := NewInternalDB(nil, nil) - dmFilterPass := NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dmFilterPass := NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := FilterS{ cfg: cfg, dm: dmFilterPass, @@ -921,9 +921,9 @@ func TestPassFiltersForEventWithEmptyFilter(t *testing.T) { } func TestPassFilterMaxCost(t *testing.T) { - data := NewInternalDB(nil, nil) - dmFilterPass := NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dmFilterPass := NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := FilterS{ cfg: cfg, dm: dmFilterPass, @@ -973,9 +973,9 @@ func TestPassFilterMaxCost(t *testing.T) { } func TestPassFilterMissingField(t *testing.T) { - data := NewInternalDB(nil, nil) - dmFilterPass := NewDataManager(data, config.CgrConfig().CacheCfg(), nil) cfg, _ := config.NewDefaultCGRConfig() + data := NewInternalDB(nil, nil, true, cfg.DataDbCfg().Items) + dmFilterPass := NewDataManager(data, config.CgrConfig().CacheCfg(), nil) filterS := FilterS{ cfg: cfg, dm: dmFilterPass, diff --git a/engine/libtest.go b/engine/libtest.go index 8aa7c036d..bfaa1d902 100644 --- a/engine/libtest.go +++ b/engine/libtest.go @@ -287,7 +287,7 @@ func InitDataDb(cfg *config.CGRConfig) error { cfg.DataDbCfg().DataDbHost, cfg.DataDbCfg().DataDbPort, cfg.DataDbCfg().DataDbName, cfg.DataDbCfg().DataDbUser, cfg.DataDbCfg().DataDbPass, cfg.GeneralCfg().DBDataEncoding, - cfg.DataDbCfg().DataDbSentinelName) + cfg.DataDbCfg().DataDbSentinelName, cfg.DataDbCfg().Items) if err != nil { return err } @@ -311,7 +311,7 @@ func InitStorDb(cfg *config.CGRConfig) error { cfg.StorDbCfg().Password, cfg.StorDbCfg().SSLMode, cfg.StorDbCfg().MaxOpenConns, cfg.StorDbCfg().MaxIdleConns, cfg.StorDbCfg().ConnMaxLifetime, cfg.StorDbCfg().StringIndexedFields, - cfg.StorDbCfg().PrefixIndexedFields) + cfg.StorDbCfg().PrefixIndexedFields, cfg.StorDbCfg().Items) if err != nil { return err } diff --git a/engine/loader_it_test.go b/engine/loader_it_test.go index a277428b8..14425b4fa 100644 --- a/engine/loader_it_test.go +++ b/engine/loader_it_test.go @@ -46,7 +46,7 @@ func TestLoaderITConnDataDbs(t *testing.T) { dbConn, err := NewDataDBConn(lCfg.DataDbCfg().DataDbType, lCfg.DataDbCfg().DataDbHost, lCfg.DataDbCfg().DataDbPort, "7", lCfg.DataDbCfg().DataDbUser, lCfg.DataDbCfg().DataDbPass, - lCfg.GeneralCfg().DBDataEncoding, "") + lCfg.GeneralCfg().DBDataEncoding, "", lCfg.DataDbCfg().Items) if err != nil { t.Fatal("Error on dataDb connection: ", err.Error()) } @@ -54,7 +54,7 @@ func TestLoaderITConnDataDbs(t *testing.T) { dbConn, err = NewDataDBConn(lCfg.DataDbCfg().DataDbType, lCfg.DataDbCfg().DataDbHost, lCfg.DataDbCfg().DataDbPort, "8", lCfg.DataDbCfg().DataDbUser, lCfg.DataDbCfg().DataDbPass, - lCfg.GeneralCfg().DBDataEncoding, "") + lCfg.GeneralCfg().DBDataEncoding, "", lCfg.DataDbCfg().Items) if err != nil { t.Fatal("Error on dataDb connection: ", err.Error()) } @@ -62,7 +62,7 @@ func TestLoaderITConnDataDbs(t *testing.T) { dbConn, err = NewDataDBConn(lCfg.DataDbCfg().DataDbType, lCfg.DataDbCfg().DataDbHost, lCfg.DataDbCfg().DataDbPort, "9", lCfg.DataDbCfg().DataDbUser, lCfg.DataDbCfg().DataDbPass, - lCfg.GeneralCfg().DBDataEncoding, "") + lCfg.GeneralCfg().DBDataEncoding, "", lCfg.DataDbCfg().Items) if err != nil { t.Fatal("Error on dataDb connection: ", err.Error()) } diff --git a/engine/onstor_it_test.go b/engine/onstor_it_test.go index a93556247..cad131bc6 100644 --- a/engine/onstor_it_test.go +++ b/engine/onstor_it_test.go @@ -120,7 +120,7 @@ func TestOnStorITMongo(t *testing.T) { func TestOnStorITInternal(t *testing.T) { sleepDelay = 10 * time.Millisecond - onStor = NewDataManager(NewInternalDB(nil, nil), + onStor = NewDataManager(NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items), config.CgrConfig().CacheCfg(), nil) for _, stest := range sTestsOnStorIT { t.Run("TestOnStorITInternal", stest) diff --git a/engine/resources_test.go b/engine/resources_test.go index 9ed0415da..f827f3485 100644 --- a/engine/resources_test.go +++ b/engine/resources_test.go @@ -363,12 +363,9 @@ func TestRSCacheSetGet(t *testing.T) { } func TestResourcePopulateResourceService(t *testing.T) { - data := NewInternalDB(nil, nil) + defaultCfg, _ := config.NewDefaultCGRConfig() + data := NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) dmRES = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - defaultCfg, err := config.NewDefaultCGRConfig() - if err != nil { - t.Errorf("Error: %+v", err) - } defaultCfg.ResourceSCfg().StoreInterval = 1 defaultCfg.ResourceSCfg().StringIndexedFields = nil defaultCfg.ResourceSCfg().PrefixIndexedFields = nil @@ -725,12 +722,10 @@ func TestResourceCaching(t *testing.T) { //clear the cache Cache.Clear(nil) // start fresh with new dataManager - data := NewInternalDB(nil, nil) + defaultCfg, _ := config.NewDefaultCGRConfig() + data := NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) dmRES = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - defaultCfg, err := config.NewDefaultCGRConfig() - if err != nil { - t.Errorf("Error: %+v", err) - } + defaultCfg.ResourceSCfg().StoreInterval = 1 defaultCfg.ResourceSCfg().StringIndexedFields = nil defaultCfg.ResourceSCfg().PrefixIndexedFields = nil diff --git a/engine/stats_test.go b/engine/stats_test.go index fd72abcf0..d8973be42 100644 --- a/engine/stats_test.go +++ b/engine/stats_test.go @@ -142,12 +142,9 @@ var ( ) func TestStatQueuesPopulateService(t *testing.T) { - data := NewInternalDB(nil, nil) + defaultCfg, _ := config.NewDefaultCGRConfig() + data := NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) dmSTS = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - defaultCfg, err := config.NewDefaultCGRConfig() - if err != nil { - t.Errorf("Error: %+v", err) - } defaultCfg.StatSCfg().StoreInterval = 1 defaultCfg.StatSCfg().StringIndexedFields = nil diff --git a/engine/storage_cdrs_it_test.go b/engine/storage_cdrs_it_test.go index 1e878d20b..a62acfa0d 100644 --- a/engine/storage_cdrs_it_test.go +++ b/engine/storage_cdrs_it_test.go @@ -106,7 +106,7 @@ func testSetCDR(cfg *config.CGRConfig) error { cfg.StorDbCfg().Password, cfg.StorDbCfg().SSLMode, cfg.StorDbCfg().MaxOpenConns, cfg.StorDbCfg().MaxIdleConns, cfg.StorDbCfg().ConnMaxLifetime, cfg.StorDbCfg().StringIndexedFields, - cfg.StorDbCfg().PrefixIndexedFields) + cfg.StorDbCfg().PrefixIndexedFields, cfg.StorDbCfg().Items) if err != nil { return err } @@ -213,7 +213,7 @@ func testSMCosts(cfg *config.CGRConfig) error { cfg.StorDbCfg().Password, cfg.StorDbCfg().SSLMode, cfg.StorDbCfg().MaxOpenConns, cfg.StorDbCfg().MaxIdleConns, cfg.StorDbCfg().ConnMaxLifetime, cfg.StorDbCfg().StringIndexedFields, - cfg.StorDbCfg().PrefixIndexedFields) + cfg.StorDbCfg().PrefixIndexedFields, cfg.StorDbCfg().Items) if err != nil { return fmt.Errorf("testSMCosts #2 err: %v", err) } @@ -276,7 +276,7 @@ func testGetCDRs(cfg *config.CGRConfig) error { cfg.StorDbCfg().Password, cfg.StorDbCfg().SSLMode, cfg.StorDbCfg().MaxOpenConns, cfg.StorDbCfg().MaxIdleConns, cfg.StorDbCfg().ConnMaxLifetime, cfg.StorDbCfg().StringIndexedFields, - cfg.StorDbCfg().PrefixIndexedFields) + cfg.StorDbCfg().PrefixIndexedFields, cfg.StorDbCfg().Items) if err != nil { return fmt.Errorf("testGetCDRs #2: %v", err) } diff --git a/engine/storage_internal_datadb.go b/engine/storage_internal_datadb.go index d99053c09..8a1b54763 100644 --- a/engine/storage_internal_datadb.go +++ b/engine/storage_internal_datadb.go @@ -31,223 +31,231 @@ import ( ) // internalDBCacheCfg indexes the internal DataDB partitions -var internalDBCacheCfg = map[string]*ltcache.CacheConfig{ - utils.CacheDestinations: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheDestinations].TTL, - }, - utils.CacheReverseDestinations: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheReverseDestinations].TTL, - }, - utils.CacheActions: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheActions].TTL, - }, - utils.CacheActionPlans: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheActionPlans].TTL, - }, - utils.CacheAccountActionPlans: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheAccountActionPlans].TTL, - }, - utils.CacheActionTriggers: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheActionTriggers].TTL, - }, - utils.CacheRatingPlans: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheRatingPlans].TTL, - }, - utils.CacheRatingProfiles: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheRatingProfiles].TTL, - }, - utils.CacheAccounts: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheAccounts].TTL, - }, - utils.CacheSharedGroups: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheSharedGroups].TTL, - }, - utils.TBLVersions: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLVersions].TTL, - }, - utils.CacheTimings: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheTimings].TTL, - }, - utils.CacheFilters: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheFilters].TTL, - }, - utils.CacheResourceProfiles: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheResourceProfiles].TTL, - }, - utils.CacheResourceFilterIndexes: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.MetaFilterIndexes].TTL, - }, - utils.CacheResources: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheResources].TTL, - }, - utils.CacheStatFilterIndexes: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.MetaFilterIndexes].TTL, - }, - utils.CacheStatQueueProfiles: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheStatQueueProfiles].TTL, - }, - utils.CacheStatQueues: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheStatQueues].TTL, - }, - utils.CacheThresholdFilterIndexes: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.MetaFilterIndexes].TTL, - }, - utils.CacheThresholdProfiles: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheThresholdProfiles].TTL, - }, - utils.CacheThresholds: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheThresholds].TTL, - }, - utils.CacheSupplierFilterIndexes: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.MetaFilterIndexes].TTL, - }, - utils.CacheSupplierProfiles: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheSupplierProfiles].TTL, - }, - utils.CacheChargerFilterIndexes: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.MetaFilterIndexes].TTL, - }, - utils.CacheChargerProfiles: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheChargerProfiles].TTL, - }, - utils.CacheAttributeFilterIndexes: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.MetaFilterIndexes].TTL, - }, - utils.CacheAttributeProfiles: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheAttributeProfiles].TTL, - }, - utils.CacheDispatcherFilterIndexes: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.MetaFilterIndexes].TTL, - }, - utils.CacheDispatcherProfiles: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheDispatcherProfiles].TTL, - }, - utils.CacheDispatcherHosts: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheDispatcherHosts].TTL, - }, - utils.CacheLoadIDs: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().DataDbCfg().Items[utils.CacheLoadIDs].TTL, - }, - utils.TBLTPTimings: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPTimings].TTL, - }, - utils.TBLTPDestinations: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPDestinations].TTL, - }, - utils.TBLTPRates: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPRates].TTL, - }, - utils.TBLTPDestinationRates: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPDestinationRates].TTL, - }, - utils.TBLTPRatingPlans: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPRatingPlans].TTL, - }, - utils.TBLTPRateProfiles: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPRateProfiles].TTL, - }, - utils.TBLTPSharedGroups: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPSharedGroups].TTL, - }, - utils.TBLTPActions: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPActions].TTL, - }, - utils.TBLTPActionTriggers: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPActionTriggers].TTL, - }, - utils.TBLTPAccountActions: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPAccountActions].TTL, - }, - utils.TBLTPResources: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPResources].TTL, - }, - utils.TBLTPStats: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPStats].TTL, - }, - utils.TBLTPThresholds: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPThresholds].TTL, - }, - utils.TBLTPFilters: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPFilters].TTL, - }, - utils.SessionCostsTBL: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.SessionCostsTBL].TTL, - }, - utils.TBLTPActionPlans: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPActionPlans].TTL, - }, - utils.TBLTPSuppliers: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPSuppliers].TTL, - }, - utils.TBLTPAttributes: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPAttributes].TTL, - }, - utils.TBLTPChargers: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPChargers].TTL, - }, - utils.TBLTPDispatchers: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPDispatchers].TTL, - }, - utils.TBLTPDispatcherHosts: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.TBLTPDispatcherHosts].TTL, - }, - utils.CDRsTBL: <cache.CacheConfig{ - MaxItems: -1, - TTL: config.CgrConfig().StorDbCfg().Items[utils.CDRsTBL].TTL, - }, +func newInternalDBCfg(itemsCacheCfg map[string]*config.ItemOpt, isDataDB bool) map[string]*ltcache.CacheConfig { + if isDataDB { + return map[string]*ltcache.CacheConfig{ + utils.CacheDestinations: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheDestinations].TTL, + }, + utils.CacheReverseDestinations: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheReverseDestinations].TTL, + }, + utils.CacheActions: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheActions].TTL, + }, + utils.CacheActionPlans: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheActionPlans].TTL, + }, + utils.CacheAccountActionPlans: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheAccountActionPlans].TTL, + }, + utils.CacheActionTriggers: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheActionTriggers].TTL, + }, + utils.CacheRatingPlans: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheRatingPlans].TTL, + }, + utils.CacheRatingProfiles: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheRatingProfiles].TTL, + }, + utils.CacheAccounts: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheAccounts].TTL, + }, + utils.CacheSharedGroups: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheSharedGroups].TTL, + }, + + utils.CacheTimings: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheTimings].TTL, + }, + utils.CacheFilters: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheFilters].TTL, + }, + utils.CacheResourceProfiles: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheResourceProfiles].TTL, + }, + utils.CacheResourceFilterIndexes: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.MetaFilterIndexes].TTL, + }, + utils.CacheResources: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheResources].TTL, + }, + utils.CacheStatFilterIndexes: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.MetaFilterIndexes].TTL, + }, + utils.CacheStatQueueProfiles: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheStatQueueProfiles].TTL, + }, + utils.CacheStatQueues: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheStatQueues].TTL, + }, + utils.CacheThresholdFilterIndexes: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.MetaFilterIndexes].TTL, + }, + utils.CacheThresholdProfiles: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheThresholdProfiles].TTL, + }, + utils.CacheThresholds: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheThresholds].TTL, + }, + utils.CacheSupplierFilterIndexes: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.MetaFilterIndexes].TTL, + }, + utils.CacheSupplierProfiles: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheSupplierProfiles].TTL, + }, + utils.CacheChargerFilterIndexes: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.MetaFilterIndexes].TTL, + }, + utils.CacheChargerProfiles: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheChargerProfiles].TTL, + }, + utils.CacheAttributeFilterIndexes: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.MetaFilterIndexes].TTL, + }, + utils.CacheAttributeProfiles: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheAttributeProfiles].TTL, + }, + utils.CacheDispatcherFilterIndexes: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.MetaFilterIndexes].TTL, + }, + utils.CacheDispatcherProfiles: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheDispatcherProfiles].TTL, + }, + utils.CacheDispatcherHosts: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheDispatcherHosts].TTL, + }, + utils.CacheLoadIDs: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CacheLoadIDs].TTL, + }, + } + } else { + return map[string]*ltcache.CacheConfig{ + utils.TBLVersions: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLVersions].TTL, + }, + utils.TBLTPTimings: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPTimings].TTL, + }, + utils.TBLTPDestinations: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPDestinations].TTL, + }, + utils.TBLTPRates: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPRates].TTL, + }, + utils.TBLTPDestinationRates: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPDestinationRates].TTL, + }, + utils.TBLTPRatingPlans: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPRatingPlans].TTL, + }, + utils.TBLTPRateProfiles: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPRateProfiles].TTL, + }, + utils.TBLTPSharedGroups: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPSharedGroups].TTL, + }, + utils.TBLTPActions: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPActions].TTL, + }, + utils.TBLTPActionTriggers: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPActionTriggers].TTL, + }, + utils.TBLTPAccountActions: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPAccountActions].TTL, + }, + utils.TBLTPResources: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPResources].TTL, + }, + utils.TBLTPStats: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPStats].TTL, + }, + utils.TBLTPThresholds: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPThresholds].TTL, + }, + utils.TBLTPFilters: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPFilters].TTL, + }, + utils.SessionCostsTBL: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.SessionCostsTBL].TTL, + }, + utils.TBLTPActionPlans: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPActionPlans].TTL, + }, + utils.TBLTPSuppliers: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPSuppliers].TTL, + }, + utils.TBLTPAttributes: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPAttributes].TTL, + }, + utils.TBLTPChargers: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPChargers].TTL, + }, + utils.TBLTPDispatchers: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPDispatchers].TTL, + }, + utils.TBLTPDispatcherHosts: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.TBLTPDispatcherHosts].TTL, + }, + utils.CDRsTBL: <cache.CacheConfig{ + MaxItems: -1, + TTL: itemsCacheCfg[utils.CDRsTBL].TTL, + }, + } + } } type InternalDB struct { @@ -261,9 +269,10 @@ type InternalDB struct { } // NewInternalDB constructs an InternalDB -func NewInternalDB(stringIndexedFields, prefixIndexedFields []string) (iDB *InternalDB) { +func NewInternalDB(stringIndexedFields, prefixIndexedFields []string, + isDataDB bool, itemsCacheCfg map[string]*config.ItemOpt) (iDB *InternalDB) { iDB = &InternalDB{ - db: ltcache.NewTransCache(internalDBCacheCfg), + db: ltcache.NewTransCache(newInternalDBCfg(itemsCacheCfg, isDataDB)), stringIndexedFields: stringIndexedFields, prefixIndexedFields: prefixIndexedFields, cnter: utils.NewCounter(time.Now().UnixNano(), 0), @@ -288,7 +297,7 @@ func (iDB *InternalDB) SetPrefixIndexedFields(prefixIndexedFields []string) { func (iDB *InternalDB) Close() {} func (iDB *InternalDB) Flush(_ string) error { - iDB.db = ltcache.NewTransCache(internalDBCacheCfg) + iDB.db.Clear(nil) return nil } diff --git a/engine/storage_utils.go b/engine/storage_utils.go index a20447f04..a03ada36b 100644 --- a/engine/storage_utils.go +++ b/engine/storage_utils.go @@ -24,6 +24,8 @@ import ( "strings" "time" + "github.com/cgrates/cgrates/config" + "github.com/cgrates/cgrates/utils" ) @@ -31,7 +33,8 @@ import ( // NewDataDBConn creates a DataDB connection func NewDataDBConn(dbType, host, port, name, user, - pass, marshaler, sentinelName string) (d DataDB, err error) { + pass, marshaler, sentinelName string, + itemsCacheCfg map[string]*config.ItemOpt) (d DataDB, err error) { switch dbType { case utils.REDIS: var dbNo int @@ -47,7 +50,7 @@ func NewDataDBConn(dbType, host, port, name, user, case utils.MONGO: d, err = NewMongoStorage(host, port, name, user, pass, utils.DataDB, nil, true) case utils.INTERNAL: - d = NewInternalDB(nil, nil) + d = NewInternalDB(nil, nil, true, itemsCacheCfg) default: err = fmt.Errorf("unsupported db_type <%s>", dbType) } @@ -57,7 +60,8 @@ func NewDataDBConn(dbType, host, port, name, user, // NewStorDBConn returns a StorDB(implements Storage interface) based on dbType func NewStorDBConn(dbType, host, port, name, user, pass, sslmode string, maxConn, maxIdleConn, connMaxLifetime int, - stringIndexedFields, prefixIndexedFields []string) (db StorDB, err error) { + stringIndexedFields, prefixIndexedFields []string, + itemsCacheCfg map[string]*config.ItemOpt) (db StorDB, err error) { switch dbType { case utils.MONGO: db, err = NewMongoStorage(host, port, name, user, pass, utils.StorDB, stringIndexedFields, false) @@ -66,7 +70,7 @@ func NewStorDBConn(dbType, host, port, name, user, pass, sslmode string, case utils.MYSQL: db, err = NewMySQLStorage(host, port, name, user, pass, maxConn, maxIdleConn, connMaxLifetime) case utils.INTERNAL: - db = NewInternalDB(stringIndexedFields, prefixIndexedFields) + db = NewInternalDB(stringIndexedFields, prefixIndexedFields, false, itemsCacheCfg) default: err = fmt.Errorf("unknown db '%s' valid options are [%s, %s, %s, %s]", dbType, utils.MYSQL, utils.MONGO, utils.POSTGRES, utils.INTERNAL) diff --git a/engine/stordb_it_test.go b/engine/stordb_it_test.go index 0e008cce2..6864f366e 100644 --- a/engine/stordb_it_test.go +++ b/engine/stordb_it_test.go @@ -124,7 +124,7 @@ func TestStorDBitInternalDB(t *testing.T) { t.Error(err) } config.SetCgrConfig(cfg) - storDB = NewInternalDB(nil, nil) + storDB = NewInternalDB(nil, nil, false, cfg.StorDbCfg().Items) for _, stest := range sTestsStorDBit { stestFullName := runtime.FuncForPC(reflect.ValueOf(stest).Pointer()).Name() split := strings.Split(stestFullName, ".") diff --git a/engine/suppliers_test.go b/engine/suppliers_test.go index 10305cfae..4f2bcac96 100644 --- a/engine/suppliers_test.go +++ b/engine/suppliers_test.go @@ -288,13 +288,9 @@ func TestSuppliersSort(t *testing.T) { } func TestSuppliersPopulateSupplierService(t *testing.T) { - data := NewInternalDB(nil, nil) + defaultCfg, _ := config.NewDefaultCGRConfig() + data := NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) dmSPP = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - defaultCfg, err := config.NewDefaultCGRConfig() - if err != nil { - t.Errorf("Error: %+v", err) - } - defaultCfg.SupplierSCfg().StringIndexedFields = nil defaultCfg.SupplierSCfg().PrefixIndexedFields = nil splService, err = NewSupplierService(dmSPP, &FilterS{ diff --git a/engine/thresholds_test.go b/engine/thresholds_test.go index 48692aafb..0bae470b0 100644 --- a/engine/thresholds_test.go +++ b/engine/thresholds_test.go @@ -142,12 +142,9 @@ func TestThresholdsSort(t *testing.T) { } func TestThresholdsPopulateThresholdService(t *testing.T) { - data := NewInternalDB(nil, nil) + defaultCfg, _ := config.NewDefaultCGRConfig() + data := NewInternalDB(nil, nil, true, defaultCfg.DataDbCfg().Items) dmTH = NewDataManager(data, config.CgrConfig().CacheCfg(), nil) - defaultCfg, err := config.NewDefaultCGRConfig() - if err != nil { - t.Errorf("Error: %+v", err) - } defaultCfg.ThresholdSCfg().StoreInterval = 0 defaultCfg.ThresholdSCfg().StringIndexedFields = nil defaultCfg.ThresholdSCfg().PrefixIndexedFields = nil diff --git a/engine/versions_it_test.go b/engine/versions_it_test.go index 29652c553..c618c0005 100644 --- a/engine/versions_it_test.go +++ b/engine/versions_it_test.go @@ -49,7 +49,7 @@ func TestVersionsITMongo(t *testing.T) { cfg.DataDbCfg().DataDbHost, cfg.DataDbCfg().DataDbPort, cfg.DataDbCfg().DataDbName, cfg.DataDbCfg().DataDbUser, cfg.DataDbCfg().DataDbPass, cfg.GeneralCfg().DBDataEncoding, - "") + "", cfg.DataDbCfg().Items) if err != nil { log.Fatal(err) } @@ -60,7 +60,7 @@ func TestVersionsITMongo(t *testing.T) { cfg.StorDbCfg().Password, cfg.StorDbCfg().SSLMode, cfg.StorDbCfg().MaxOpenConns, cfg.StorDbCfg().MaxIdleConns, cfg.StorDbCfg().ConnMaxLifetime, cfg.StorDbCfg().StringIndexedFields, - cfg.StorDbCfg().PrefixIndexedFields) + cfg.StorDbCfg().PrefixIndexedFields, cfg.StorDbCfg().Items) if err != nil { log.Fatal(err) } @@ -79,7 +79,7 @@ func TestVersionsITRedisMYSQL(t *testing.T) { cfg.DataDbCfg().DataDbHost, cfg.DataDbCfg().DataDbPort, cfg.DataDbCfg().DataDbName, cfg.DataDbCfg().DataDbUser, cfg.DataDbCfg().DataDbPass, cfg.GeneralCfg().DBDataEncoding, - "") + "", cfg.DataDbCfg().Items) if err != nil { log.Fatal(err) } @@ -91,7 +91,7 @@ func TestVersionsITRedisMYSQL(t *testing.T) { cfg.StorDbCfg().Password, cfg.StorDbCfg().SSLMode, cfg.StorDbCfg().MaxOpenConns, cfg.StorDbCfg().MaxIdleConns, cfg.StorDbCfg().ConnMaxLifetime, cfg.StorDbCfg().StringIndexedFields, - cfg.StorDbCfg().PrefixIndexedFields) + cfg.StorDbCfg().PrefixIndexedFields, cfg.StorDbCfg().Items) if err != nil { log.Fatal(err) } @@ -110,7 +110,7 @@ func TestVersionsITRedisPostgres(t *testing.T) { cfg.DataDbCfg().DataDbHost, cfg.DataDbCfg().DataDbPort, cfg.DataDbCfg().DataDbName, cfg.DataDbCfg().DataDbUser, cfg.DataDbCfg().DataDbPass, cfg.GeneralCfg().DBDataEncoding, - "") + "", cfg.DataDbCfg().Items) if err != nil { log.Fatal(err) } @@ -121,7 +121,7 @@ func TestVersionsITRedisPostgres(t *testing.T) { cfg.StorDbCfg().Password, cfg.StorDbCfg().SSLMode, cfg.StorDbCfg().MaxOpenConns, cfg.StorDbCfg().MaxIdleConns, cfg.StorDbCfg().ConnMaxLifetime, cfg.StorDbCfg().StringIndexedFields, - cfg.StorDbCfg().PrefixIndexedFields) + cfg.StorDbCfg().PrefixIndexedFields, cfg.StorDbCfg().Items) if err != nil { log.Fatal(err) } diff --git a/general_tests/acntacts_test.go b/general_tests/acntacts_test.go index b42723cfc..77d22cca7 100644 --- a/general_tests/acntacts_test.go +++ b/general_tests/acntacts_test.go @@ -29,7 +29,7 @@ import ( var dbAcntActs *engine.DataManager func TestAcntActsSetStorage(t *testing.T) { - dataDB := engine.NewInternalDB(nil, nil) + dataDB := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) dbAcntActs = engine.NewDataManager(dataDB, config.CgrConfig().CacheCfg(), nil) engine.SetDataStorage(dbAcntActs) } diff --git a/general_tests/auth_test.go b/general_tests/auth_test.go index 116b67125..6294150ed 100644 --- a/general_tests/auth_test.go +++ b/general_tests/auth_test.go @@ -31,7 +31,7 @@ var rsponder *engine.Responder func TestAuthSetStorage(t *testing.T) { config.CgrConfig().CacheCfg()[utils.CacheRatingPlans].Precache = true // precache rating plan - dataDB := engine.NewInternalDB(nil, nil) + dataDB := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) dbAuth = engine.NewDataManager(dataDB, config.CgrConfig().CacheCfg(), nil) engine.SetDataStorage(dbAuth) rsponder = &engine.Responder{ diff --git a/general_tests/cdrs_internal_it_test.go b/general_tests/cdrs_internal_it_test.go new file mode 100644 index 000000000..1c397cca7 --- /dev/null +++ b/general_tests/cdrs_internal_it_test.go @@ -0,0 +1,125 @@ +// +build integration + +/* +Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments +Copyright (C) ITsysCOM GmbH + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package general_tests + +import ( + "net/rpc" + "path" + "testing" + "time" + + "github.com/cgrates/cgrates/utils" + + "github.com/cgrates/cgrates/engine" + + "github.com/cgrates/cgrates/config" +) + +var ( + cdrsIntCfgPath string + cdrsIntCfg *config.CGRConfig + cdrsIntRPC *rpc.Client + + sTestsCdrsInt = []func(t *testing.T){ + testCdrsIntInitCfg, + testCdrsIntStartEngine, + testCdrsIntRpcConn, + testCdrsIntTestTTL, + testCdrsIntStopEngine, + } +) + +// This test is valid only for internal +// to test the ttl for cdrs +func TestCdrsIntITMySql(t *testing.T) { + for _, stest := range sTestsCdrsInt { + t.Run("TestCdrsIntITMySql", stest) + } +} + +func testCdrsIntInitCfg(t *testing.T) { + var err error + cdrsIntCfgPath = path.Join(cdreDataDir, "conf", "samples", "internal_ttl") + cdrsIntCfg, err = config.NewCGRConfigFromPath(cdrsIntCfgPath) + if err != nil { + t.Error(err) + } + cdrsIntCfg.DataFolderPath = cdreDataDir +} + +func testCdrsIntStartEngine(t *testing.T) { + if _, err := engine.StopStartEngine(cdrsIntCfgPath, waitRater); err != nil { + t.Fatal(err) + } +} + +func testCdrsIntRpcConn(t *testing.T) { + var err error + cdrsIntRPC, err = newRPCClient(cdrsIntCfg.ListenCfg()) + if err != nil { + t.Fatal(err) + } +} + +func testCdrsIntTestTTL(t *testing.T) { + args := &engine.ArgV1ProcessEvent{ + Flags: []string{"*store:true"}, + CGREvent: utils.CGREvent{ + Tenant: "cgrates.org", + Event: map[string]interface{}{ + utils.OriginID: "testCdrsIntTestTTL", + utils.OriginHost: "192.168.1.1", + utils.Source: "testCdrsIntTestTTL", + utils.RequestType: utils.META_NONE, + utils.Category: "call", + utils.Account: "testCdrsIntTestTTL", + utils.Subject: "ANY2CNT2", + utils.Destination: "+4986517174963", + utils.AnswerTime: time.Date(2018, 8, 24, 16, 00, 26, 0, time.UTC), + utils.Usage: time.Minute, + }, + }, + } + + var reply string + if err := cdrsIntRPC.Call(utils.CDRsV1ProcessEvent, args, &reply); err != nil { + t.Error("Unexpected error: ", err.Error()) + } else if reply != utils.OK { + t.Error("Unexpected reply received: ", reply) + } + var cdrs []*engine.ExternalCDR + if err := cdrsIntRPC.Call(utils.ApierV2GetCDRs, &utils.RPCCDRsFilter{}, &cdrs); err != nil { + t.Fatal("Unexpected error: ", err.Error()) + } else if len(cdrs) != 1 { + t.Errorf("Expected 1 result received %v ", len(cdrs)) + } + time.Sleep(3 * time.Second) + if err := cdrsIntRPC.Call(utils.ApierV2GetCDRs, &utils.RPCCDRsFilter{}, &cdrs); err == nil || + err.Error() != utils.ErrNotFound.Error() { + t.Fatal("Unexpected error: ", err.Error()) + } +} + +func testCdrsIntStopEngine(t *testing.T) { + if err := engine.KillEngine(waitRater); err != nil { + t.Error(err) + } +} diff --git a/general_tests/costs1_test.go b/general_tests/costs1_test.go index 654b7160b..ec151994f 100644 --- a/general_tests/costs1_test.go +++ b/general_tests/costs1_test.go @@ -27,7 +27,7 @@ import ( func TestCosts1SetStorage(t *testing.T) { config.CgrConfig().CacheCfg()[utils.CacheRatingPlans].Precache = true // precache rating plan - dataDBInt := engine.NewInternalDB(nil, nil) + dataDBInt := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) dataDB = engine.NewDataManager(dataDBInt, config.CgrConfig().CacheCfg(), nil) engine.SetDataStorage(dataDB) } diff --git a/general_tests/datachrg1_test.go b/general_tests/datachrg1_test.go index 522e479b0..c493765e4 100644 --- a/general_tests/datachrg1_test.go +++ b/general_tests/datachrg1_test.go @@ -27,7 +27,7 @@ import ( ) func TestSetStorageDtChrg1(t *testing.T) { - dataDBInt := engine.NewInternalDB(nil, nil) + dataDBInt := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) dataDB = engine.NewDataManager(dataDBInt, config.CgrConfig().CacheCfg(), nil) engine.SetDataStorage(dataDB) } diff --git a/general_tests/ddazmbl1_test.go b/general_tests/ddazmbl1_test.go index c2b70e747..065474c41 100644 --- a/general_tests/ddazmbl1_test.go +++ b/general_tests/ddazmbl1_test.go @@ -30,7 +30,7 @@ import ( var dataDB *engine.DataManager func TestDZ1SetStorage(t *testing.T) { - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) dataDB = engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) engine.SetDataStorage(dataDB) } diff --git a/general_tests/ddazmbl2_test.go b/general_tests/ddazmbl2_test.go index 0aadb49c8..b286acba6 100644 --- a/general_tests/ddazmbl2_test.go +++ b/general_tests/ddazmbl2_test.go @@ -30,7 +30,7 @@ import ( var dataDB2 *engine.DataManager func TestSetStorage2(t *testing.T) { - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) dataDB2 = engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) engine.SetDataStorage(dataDB2) } diff --git a/general_tests/ddazmbl3_test.go b/general_tests/ddazmbl3_test.go index 93770a0a1..4e945260b 100644 --- a/general_tests/ddazmbl3_test.go +++ b/general_tests/ddazmbl3_test.go @@ -30,7 +30,7 @@ import ( var dataDB3 *engine.DataManager func TestSetStorage3(t *testing.T) { - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) dataDB3 = engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) engine.SetDataStorage(dataDB3) } diff --git a/general_tests/smschrg1_test.go b/general_tests/smschrg1_test.go index e47f50795..5d456e9e3 100644 --- a/general_tests/smschrg1_test.go +++ b/general_tests/smschrg1_test.go @@ -28,7 +28,7 @@ import ( func TestSMSSetStorageSmsChrg1(t *testing.T) { config.CgrConfig().CacheCfg()[utils.CacheRatingPlans].Precache = true // precache rating plan - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) dataDB = engine.NewDataManager(data, config.CgrConfig().CacheCfg(), nil) engine.SetDataStorage(dataDB) } diff --git a/loaders/loader_test.go b/loaders/loader_test.go index 28ea86575..0dc8da752 100644 --- a/loaders/loader_test.go +++ b/loaders/loader_test.go @@ -33,7 +33,7 @@ import ( ) func TestLoaderProcessContentSingleFile(t *testing.T) { - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) ldr := &Loader{ ldrID: "TestLoaderProcessContent", bufLoaderData: make(map[string][]LoaderData), @@ -139,7 +139,7 @@ func TestLoaderProcessContentSingleFile(t *testing.T) { func TestLoaderProcessContentMultiFiles(t *testing.T) { file1CSV := `ignored,ignored,ignored,ignored,ignored,,Subject,1001,ignored,ignored` file2CSV := `ignored,TestLoader2` - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) ldr := &Loader{ ldrID: "TestLoaderProcessContentMultiFiles", bufLoaderData: make(map[string][]LoaderData), @@ -217,7 +217,7 @@ func TestLoaderProcessContentMultiFiles(t *testing.T) { } func TestLoaderProcessResource(t *testing.T) { - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) ldr := &Loader{ ldrID: "TestLoaderProcessResources", bufLoaderData: make(map[string][]LoaderData), @@ -335,7 +335,7 @@ func TestLoaderProcessResource(t *testing.T) { } func TestLoaderProcessFilters(t *testing.T) { - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) ldr := &Loader{ ldrID: "TestLoaderProcessFilters", bufLoaderData: make(map[string][]LoaderData), @@ -446,7 +446,7 @@ func TestLoaderProcessFilters(t *testing.T) { } func TestLoaderProcessThresholds(t *testing.T) { - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) ldr := &Loader{ ldrID: "TestLoaderProcessContent", bufLoaderData: make(map[string][]LoaderData), @@ -544,7 +544,7 @@ func TestLoaderProcessThresholds(t *testing.T) { } func TestLoaderProcessStats(t *testing.T) { - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) ldr := &Loader{ ldrID: "TestLoaderProcessContent", bufLoaderData: make(map[string][]LoaderData), @@ -666,7 +666,7 @@ func TestLoaderProcessStats(t *testing.T) { } func TestLoaderProcessSuppliers(t *testing.T) { - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) ldr := &Loader{ ldrID: "TestLoaderProcessContent", bufLoaderData: make(map[string][]LoaderData), @@ -793,7 +793,7 @@ func TestLoaderProcessSuppliers(t *testing.T) { } func TestLoaderProcessChargers(t *testing.T) { - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) ldr := &Loader{ ldrID: "TestLoaderProcessContent", bufLoaderData: make(map[string][]LoaderData), @@ -871,7 +871,7 @@ func TestLoaderProcessChargers(t *testing.T) { } func TestLoaderProcessDispatches(t *testing.T) { - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) ldr := &Loader{ ldrID: "TestLoaderProcessContent", bufLoaderData: make(map[string][]LoaderData), @@ -1023,7 +1023,7 @@ func TestLoaderProcessDispatches(t *testing.T) { } func TestLoaderProcessDispatcheHosts(t *testing.T) { - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) ldr := &Loader{ ldrID: "TestLoaderProcessContent", bufLoaderData: make(map[string][]LoaderData), @@ -1111,7 +1111,7 @@ func TestLoaderProcessDispatcheHosts(t *testing.T) { } func TestLoaderRemoveContentSingleFile(t *testing.T) { - data := engine.NewInternalDB(nil, nil) + data := engine.NewInternalDB(nil, nil, true, config.CgrConfig().DataDbCfg().Items) ldr := &Loader{ ldrID: "TestLoaderProcessContent", bufLoaderData: make(map[string][]LoaderData), diff --git a/migrator/migrator_utils.go b/migrator/migrator_utils.go index a74c00d77..9b5ff0a15 100644 --- a/migrator/migrator_utils.go +++ b/migrator/migrator_utils.go @@ -27,11 +27,12 @@ import ( "github.com/cgrates/cgrates/utils" ) -func NewMigratorDataDB(db_type, host, port, name, user, pass, marshaler string, - cacheCfg config.CacheCfg, sentinelName string) (db MigratorDataDB, err error) { +func NewMigratorDataDB(db_type, host, port, name, user, pass, + marshaler string, cacheCfg config.CacheCfg, sentinelName string, + itemsCacheCfg map[string]*config.ItemOpt) (db MigratorDataDB, err error) { dbCon, err := engine.NewDataDBConn(db_type, host, port, name, user, pass, marshaler, - sentinelName) + sentinelName, itemsCacheCfg) if err != nil { return nil, err } @@ -54,11 +55,12 @@ func NewMigratorDataDB(db_type, host, port, name, user, pass, marshaler string, } func NewMigratorStorDB(db_type, host, port, name, user, pass, sslmode string, - maxConn, maxIdleConn, connMaxLifetime int, - stringIndexedFields, prefixIndexedFields []string) (db MigratorStorDB, err error) { + maxConn, maxIdleConn, connMaxLifetime int, stringIndexedFields, prefixIndexedFields []string, + itemsCacheCfg map[string]*config.ItemOpt) (db MigratorStorDB, err error) { var d MigratorStorDB - storDb, err := engine.NewStorDBConn(db_type, host, port, name, user, pass, sslmode, - maxConn, maxIdleConn, connMaxLifetime, stringIndexedFields, prefixIndexedFields) + storDb, err := engine.NewStorDBConn(db_type, host, port, name, user, + pass, sslmode, maxConn, maxIdleConn, connMaxLifetime, + stringIndexedFields, prefixIndexedFields, itemsCacheCfg) if err != nil { return nil, err } diff --git a/services/datadb.go b/services/datadb.go index 2fadb2a1a..c38ab683c 100644 --- a/services/datadb.go +++ b/services/datadb.go @@ -60,7 +60,7 @@ func (db *DataDBService) Start() (err error) { db.cfg.DataDbCfg().DataDbHost, db.cfg.DataDbCfg().DataDbPort, db.cfg.DataDbCfg().DataDbName, db.cfg.DataDbCfg().DataDbUser, db.cfg.DataDbCfg().DataDbPass, db.cfg.GeneralCfg().DBDataEncoding, - db.cfg.DataDbCfg().DataDbSentinelName) + db.cfg.DataDbCfg().DataDbSentinelName, db.cfg.DataDbCfg().Items) if db.mandatoryDB() && err != nil { // Cannot configure getter database, show stopper utils.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err)) return diff --git a/services/stordb.go b/services/stordb.go index 6f1edded7..6c47f1e75 100644 --- a/services/stordb.go +++ b/services/stordb.go @@ -61,7 +61,8 @@ func (db *StorDBService) Start() (err error) { db.cfg.StorDbCfg().Port, db.cfg.StorDbCfg().Name, db.cfg.StorDbCfg().User, db.cfg.StorDbCfg().Password, db.cfg.StorDbCfg().SSLMode, db.cfg.StorDbCfg().MaxOpenConns, db.cfg.StorDbCfg().MaxIdleConns, db.cfg.StorDbCfg().ConnMaxLifetime, - db.cfg.StorDbCfg().StringIndexedFields, db.cfg.StorDbCfg().PrefixIndexedFields) + db.cfg.StorDbCfg().StringIndexedFields, db.cfg.StorDbCfg().PrefixIndexedFields, + db.cfg.StorDbCfg().Items) if err != nil { // Cannot configure getter database, show stopper utils.Logger.Crit(fmt.Sprintf("Could not configure storDB: %s exiting!", err)) return @@ -91,7 +92,8 @@ func (db *StorDBService) Reload() (err error) { db.cfg.StorDbCfg().Port, db.cfg.StorDbCfg().Name, db.cfg.StorDbCfg().User, db.cfg.StorDbCfg().Password, db.cfg.StorDbCfg().SSLMode, db.cfg.StorDbCfg().MaxOpenConns, db.cfg.StorDbCfg().MaxIdleConns, db.cfg.StorDbCfg().ConnMaxLifetime, - db.cfg.StorDbCfg().StringIndexedFields, db.cfg.StorDbCfg().PrefixIndexedFields); err != nil { + db.cfg.StorDbCfg().StringIndexedFields, db.cfg.StorDbCfg().PrefixIndexedFields, + db.cfg.StorDbCfg().Items); err != nil { return } db.db.Close() diff --git a/sessions/sessions_test.go b/sessions/sessions_test.go index 072f7172d..b621f3f05 100644 --- a/sessions/sessions_test.go +++ b/sessions/sessions_test.go @@ -1569,7 +1569,7 @@ func TestSessionSNewV1AuthorizeArgsWithArgDispatcher2(t *testing.T) { func TestSessionSGetIndexedFilters(t *testing.T) { sSCfg, _ := config.NewDefaultCGRConfig() - mpStr := engine.NewInternalDB(nil, nil) + mpStr := engine.NewInternalDB(nil, nil, true, sSCfg.DataDbCfg().Items) sS := NewSessionS(sSCfg, engine.NewDataManager(mpStr, config.CgrConfig().CacheCfg(), nil), nil) expIndx := map[string][]string{} expUindx := []*engine.FilterRule{