Added goconf in glide, updated cgr-loader with config_path flag fixes #964

This commit is contained in:
Anevo
2018-04-25 10:40:09 -04:00
committed by Dan Christian Bogos
parent 0fe58993e8
commit 28de5ac30f
7 changed files with 246 additions and 132 deletions

View File

@@ -23,7 +23,7 @@ import (
"fmt"
"log"
"path"
"strings"
//"strings"
"time"
"github.com/cgrates/cgrates/config"
@@ -33,25 +33,25 @@ import (
)
var (
datadb_type = flag.String("datadb_type", config.CgrConfig().DataDbType, "The type of the DataDb database <*redis|*mongo>")
datadb_host = flag.String("datadb_host", utils.MetaDynamic, "The DataDb host to connect to.")
datadb_port = flag.String("datadb_port", utils.MetaDynamic, "The DataDb port to bind to.")
datadb_name = flag.String("datadb_name", utils.MetaDynamic, "The name/number of the DataDb to connect to.")
datadb_user = flag.String("datadb_user", utils.MetaDynamic, "The DataDb user to sign in as.")
datadb_pass = flag.String("datadb_passwd", utils.MetaDynamic, "The DataDb user's password.")
datadb_type = flag.String("datadb_type", "", "The type of the DataDb database <*redis|*mongo>")
datadb_host = flag.String("datadb_host", "", "The DataDb host to connect to.")
datadb_port = flag.String("datadb_port", "", "The DataDb port to bind to.")
datadb_name = flag.String("datadb_name", "", "The name/number of the DataDb to connect to.")
datadb_user = flag.String("datadb_user", "", "The DataDb user to sign in as.")
datadb_pass = flag.String("datadb_passwd", "", "The DataDb user's password.")
stor_db_type = flag.String("stordb_type", config.CgrConfig().StorDBType, "The type of the storDb database <*mysql|*postgres|*mongo>")
stor_db_host = flag.String("stordb_host", utils.MetaDynamic, "The storDb host to connect to.")
stor_db_port = flag.String("stordb_port", utils.MetaDynamic, "The storDb port to bind to.")
stor_db_name = flag.String("stordb_name", utils.MetaDynamic, "The name/number of the storDb to connect to.")
stor_db_user = flag.String("stordb_user", utils.MetaDynamic, "The storDb user to sign in as.")
stor_db_pass = flag.String("stordb_passwd", utils.MetaDynamic, "The storDb user's password.")
stor_db_type = flag.String("stordb_type", "", "The type of the storDb database <*mysql|*postgres|*mongo>")
stor_db_host = flag.String("stordb_host", "", "The storDb host to connect to.")
stor_db_port = flag.String("stordb_port", "", "The storDb port to bind to.")
stor_db_name = flag.String("stordb_name", "", "The name/number of the storDb to connect to.")
stor_db_user = flag.String("stordb_user", "", "The storDb user to sign in as.")
stor_db_pass = flag.String("stordb_passwd", "", "The storDb user's password.")
dbdata_encoding = flag.String("dbdata_encoding", config.CgrConfig().DBDataEncoding, "The encoding used to store object data in strings")
flush = flag.Bool("flushdb", false, "Flush the database before importing")
tpid = flag.String("tpid", "", "The tariff plan id from the database")
dataPath = flag.String("path", "./", "The path to folder containing the data files")
dataPath = flag.String("path", "", "The path to folder containing the data files")
version = flag.Bool("version", false, "Prints the application version.")
verbose = flag.Bool("verbose", false, "Enable detailed verbose logging output")
dryRun = flag.Bool("dry_run", false, "When true will not save loaded data to dataDb but just parse it for consistency and errors.")
@@ -59,16 +59,17 @@ var (
stats = flag.Bool("stats", false, "Generates statsistics about given data.")
fromStorDb = flag.Bool("from_stordb", false, "Load the tariff plan from storDb to dataDb")
toStorDb = flag.Bool("to_stordb", false, "Import the tariff plan from files to storDb")
rpcEncoding = flag.String("rpc_encoding", "json", "RPC encoding used <gob|json>")
ralsAddress = flag.String("rals", config.CgrConfig().RPCJSONListen, "Rater service to contact for cache reloads, empty to disable automatic cache reloads")
cdrstatsAddress = flag.String("cdrstats", config.CgrConfig().RPCJSONListen, "CDRStats service to contact for data reloads, empty to disable automatic data reloads")
usersAddress = flag.String("users", config.CgrConfig().RPCJSONListen, "Users service to contact for data reloads, empty to disable automatic data reloads")
rpcEncoding = flag.String("rpc_encoding", "", "RPC encoding used <gob|json>")
ralsAddress = flag.String("rals", "", "Rater service to contact for cache reloads, empty to disable automatic cache reloads")
cdrstatsAddress = flag.String("cdrstats", "", "CDRStats service to contact for data reloads, empty to disable automatic data reloads")
usersAddress = flag.String("users", "", "Users service to contact for data reloads, empty to disable automatic data reloads")
runId = flag.String("runid", "", "Uniquely identify an import/load, postpended to some automatic fields")
loadHistorySize = flag.Int("load_history_size", config.CgrConfig().LoadHistorySize, "Limit the number of records in the load history")
timezone = flag.String("timezone", config.CgrConfig().DefaultTimezone, `Timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB>`)
loadHistorySize = flag.Int("load_history_size", 0, "Limit the number of records in the load history")
timezone = flag.String("timezone", "", `Timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB>`)
disable_reverse = flag.Bool("disable_reverse_mappings", false, "Will disable reverse mappings rebuilding")
flush_stordb = flag.Bool("flush_stordb", false, "Remove tariff plan data for id from the database")
remove = flag.Bool("remove", false, "Will remove any data from db that matches data files")
config_path = flag.String("config_path", "/etc/cgrates/cgr-loader.cfg", "Full path towards configuration file")
)
func main() {
@@ -77,32 +78,115 @@ func main() {
fmt.Println(utils.GetCGRVersion())
return
}
var errDataDB, errStorDb, err error
var dm *engine.DataManager
var storDb engine.LoadStorage
var rater, cdrstats, users rpcclient.RpcClientConnection
var loader engine.LoadReader
*datadb_type = strings.TrimPrefix(*datadb_type, "*")
*datadb_host = config.DBDefaults.DBHost(*datadb_type, *datadb_host)
*datadb_port = config.DBDefaults.DBPort(*datadb_type, *datadb_port)
*datadb_name = config.DBDefaults.DBName(*datadb_type, *datadb_name)
*datadb_user = config.DBDefaults.DBUser(*datadb_type, *datadb_user)
*datadb_pass = config.DBDefaults.DBPass(*datadb_type, *datadb_pass)
lCfg, err := config.NewLoaderConfig(*config_path)
if err != nil {
fmt.Printf("Error loading config file %+v", err)
}
fmt.Printf("Config before changing DataDBType: %+v \n \n", utils.ToJSON(lCfg))
*stor_db_type = strings.TrimPrefix(*stor_db_type, "*")
*stor_db_host = config.DBDefaults.DBHost(*stor_db_type, *stor_db_host)
*stor_db_port = config.DBDefaults.DBPort(*stor_db_type, *stor_db_port)
*stor_db_name = config.DBDefaults.DBName(*stor_db_type, *stor_db_name)
*stor_db_user = config.DBDefaults.DBUser(*stor_db_type, *stor_db_user)
*stor_db_pass = config.DBDefaults.DBPass(*stor_db_type, *stor_db_pass)
if *datadb_type != "" {
lCfg.DataDBType = *datadb_type
}
if *datadb_host != "" {
lCfg.DataDBHost = *datadb_host
}
if *datadb_port != "" {
lCfg.DataDBPort = *datadb_port
}
if *datadb_name != "" {
lCfg.DataDBName = *datadb_name
}
if *datadb_user != "" {
lCfg.DataDBUser = *datadb_user
}
if *datadb_pass != "" {
lCfg.DataDBPass = *datadb_pass
}
if *stor_db_type != "" {
lCfg.StorDBType = *stor_db_type
}
if *stor_db_host != "" {
lCfg.StorDBHost = *stor_db_host
}
if *stor_db_port != "" {
lCfg.StorDBPort = *stor_db_port
}
if *stor_db_name != "" {
lCfg.StorDBName = *stor_db_name
}
if *stor_db_user != "" {
lCfg.StorDBUser = *stor_db_user
}
if *stor_db_pass != "" {
lCfg.StorDBPass = *stor_db_pass
}
if *loadHistorySize != 0 {
lCfg.LoadHistorySize = *loadHistorySize
}
if *tpid != "" {
lCfg.Tpid = *tpid
}
if *dataPath != "" {
lCfg.DataPath = *dataPath
}
if *rpcEncoding != "" {
lCfg.RpcEncoding = *rpcEncoding
}
if *ralsAddress != "" {
lCfg.RalsAddress = *ralsAddress
}
if *cdrstatsAddress != "" {
lCfg.CdrstatsAddress = *cdrstatsAddress
}
if *usersAddress != "" {
lCfg.UsersAddress = *usersAddress
}
if *runId != "" {
lCfg.RunId = *runId
}
if *timezone != "" {
lCfg.Timezone = *timezone
}
if *disable_reverse != false {
lCfg.DisableReverse = *disable_reverse
}
fmt.Printf("Config after changing DataDBType: %+v \n \n", utils.ToJSON(lCfg))
if !*toStorDb {
dm, errDataDB = engine.ConfigureDataStorage(*datadb_type, *datadb_host, *datadb_port, *datadb_name,
*datadb_user, *datadb_pass, *dbdata_encoding, config.CgrConfig().CacheCfg(), *loadHistorySize)
dm, errDataDB = engine.ConfigureDataStorage(lCfg.DataDBType, lCfg.DataDBHost, lCfg.DataDBPort, lCfg.DataDBName,
lCfg.DataDBUser, lCfg.DataDBPass, *dbdata_encoding, config.CgrConfig().CacheCfg(), lCfg.LoadHistorySize)
}
if *fromStorDb || *toStorDb {
storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding,
storDb, errStorDb = engine.ConfigureLoadStorage(lCfg.StorDBType, lCfg.StorDBHost, lCfg.StorDBPort, lCfg.StorDBName, lCfg.StorDBUser, lCfg.StorDBPass, *dbdata_encoding,
config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes)
}
// Stop on db errors
@@ -122,21 +206,21 @@ func main() {
//tpid_remove
if *toStorDb { // Import files from a directory into storDb
if *tpid == "" {
if lCfg.Tpid == "" {
log.Fatal("TPid required, please define it via *-tpid* command argument.")
}
if *flush_stordb {
if err = storDb.RemTpData("", *tpid, map[string]string{}); err != nil {
if err = storDb.RemTpData("", lCfg.Tpid, map[string]string{}); err != nil {
log.Fatal(err)
}
}
csvImporter := engine.TPCSVImporter{
TPid: *tpid,
TPid: lCfg.Tpid,
StorDb: storDb,
DirPath: *dataPath,
DirPath: lCfg.DataPath,
Sep: ',',
Verbose: *verbose,
ImportId: *runId,
ImportId: lCfg.RunId,
}
if errImport := csvImporter.Run(); errImport != nil {
log.Fatal(errImport)
@@ -154,32 +238,32 @@ func main() {
}
}*/
loader = engine.NewFileCSVStorage(',',
path.Join(*dataPath, utils.DESTINATIONS_CSV),
path.Join(*dataPath, utils.TIMINGS_CSV),
path.Join(*dataPath, utils.RATES_CSV),
path.Join(*dataPath, utils.DESTINATION_RATES_CSV),
path.Join(*dataPath, utils.RATING_PLANS_CSV),
path.Join(*dataPath, utils.RATING_PROFILES_CSV),
path.Join(*dataPath, utils.SHARED_GROUPS_CSV),
path.Join(*dataPath, utils.LCRS_CSV),
path.Join(*dataPath, utils.ACTIONS_CSV),
path.Join(*dataPath, utils.ACTION_PLANS_CSV),
path.Join(*dataPath, utils.ACTION_TRIGGERS_CSV),
path.Join(*dataPath, utils.ACCOUNT_ACTIONS_CSV),
path.Join(*dataPath, utils.DERIVED_CHARGERS_CSV),
path.Join(*dataPath, utils.CDR_STATS_CSV),
path.Join(*dataPath, utils.USERS_CSV),
path.Join(*dataPath, utils.ALIASES_CSV),
path.Join(*dataPath, utils.ResourcesCsv),
path.Join(*dataPath, utils.StatsCsv),
path.Join(*dataPath, utils.ThresholdsCsv),
path.Join(*dataPath, utils.FiltersCsv),
path.Join(*dataPath, utils.SuppliersCsv),
path.Join(*dataPath, utils.AttributesCsv),
path.Join(lCfg.DataPath, utils.DESTINATIONS_CSV),
path.Join(lCfg.DataPath, utils.TIMINGS_CSV),
path.Join(lCfg.DataPath, utils.RATES_CSV),
path.Join(lCfg.DataPath, utils.DESTINATION_RATES_CSV),
path.Join(lCfg.DataPath, utils.RATING_PLANS_CSV),
path.Join(lCfg.DataPath, utils.RATING_PROFILES_CSV),
path.Join(lCfg.DataPath, utils.SHARED_GROUPS_CSV),
path.Join(lCfg.DataPath, utils.LCRS_CSV),
path.Join(lCfg.DataPath, utils.ACTIONS_CSV),
path.Join(lCfg.DataPath, utils.ACTION_PLANS_CSV),
path.Join(lCfg.DataPath, utils.ACTION_TRIGGERS_CSV),
path.Join(lCfg.DataPath, utils.ACCOUNT_ACTIONS_CSV),
path.Join(lCfg.DataPath, utils.DERIVED_CHARGERS_CSV),
path.Join(lCfg.DataPath, utils.CDR_STATS_CSV),
path.Join(lCfg.DataPath, utils.USERS_CSV),
path.Join(lCfg.DataPath, utils.ALIASES_CSV),
path.Join(lCfg.DataPath, utils.ResourcesCsv),
path.Join(lCfg.DataPath, utils.StatsCsv),
path.Join(lCfg.DataPath, utils.ThresholdsCsv),
path.Join(lCfg.DataPath, utils.FiltersCsv),
path.Join(lCfg.DataPath, utils.SuppliersCsv),
path.Join(lCfg.DataPath, utils.AttributesCsv),
)
}
tpReader := engine.NewTpReader(dm.DataDB(), loader, *tpid, *timezone)
tpReader := engine.NewTpReader(dm.DataDB(), loader, *tpid, lCfg.Timezone)
err = tpReader.LoadAll()
if err != nil {
log.Fatal(err)
@@ -195,21 +279,21 @@ func main() {
if *dryRun { // We were just asked to parse the data, not saving it
return
}
if *ralsAddress != "" { // Init connection to rater so we can reload it's data
if rater, err = rpcclient.NewRpcClient("tcp", *ralsAddress, 3, 3,
time.Duration(1*time.Second), time.Duration(5*time.Minute), *rpcEncoding, nil, false); err != nil {
if lCfg.RalsAddress != "" { // Init connection to rater so we can reload it's data
if rater, err = rpcclient.NewRpcClient("tcp", lCfg.RalsAddress, 3, 3,
time.Duration(1*time.Second), time.Duration(5*time.Minute), lCfg.RpcEncoding, nil, false); err != nil {
log.Fatalf("Could not connect to RALs: %s", err.Error())
return
}
} else {
log.Print("WARNING: Rates automatic cache reloading is disabled!")
}
if *cdrstatsAddress != "" { // Init connection to rater so we can reload it's data
if *cdrstatsAddress == *ralsAddress {
if lCfg.CdrstatsAddress != "" { // Init connection to rater so we can reload it's data
if lCfg.CdrstatsAddress == lCfg.RalsAddress {
cdrstats = rater
} else {
if cdrstats, err = rpcclient.NewRpcClient("tcp", *cdrstatsAddress, 3, 3,
time.Duration(1*time.Second), time.Duration(5*time.Minute), *rpcEncoding, nil, false); err != nil {
if cdrstats, err = rpcclient.NewRpcClient("tcp", lCfg.CdrstatsAddress, 3, 3,
time.Duration(1*time.Second), time.Duration(5*time.Minute), lCfg.RpcEncoding, nil, false); err != nil {
log.Fatalf("Could not connect to CDRStatS API: %s", err.Error())
return
}
@@ -217,12 +301,12 @@ func main() {
} else {
log.Print("WARNING: CDRStats automatic data reload is disabled!")
}
if *usersAddress != "" { // Init connection to rater so we can reload it's data
if *usersAddress == *ralsAddress {
if lCfg.UsersAddress != "" { // Init connection to rater so we can reload it's data
if lCfg.UsersAddress == lCfg.RalsAddress {
users = rater
} else {
if users, err = rpcclient.NewRpcClient("tcp", *usersAddress, 3, 3,
time.Duration(1*time.Second), time.Duration(5*time.Minute), *rpcEncoding, nil, false); err != nil {
if users, err = rpcclient.NewRpcClient("tcp", lCfg.UsersAddress, 3, 3,
time.Duration(1*time.Second), time.Duration(5*time.Minute), lCfg.RpcEncoding, nil, false); err != nil {
log.Fatalf("Could not connect to UserS API: %s", err.Error())
return
}
@@ -232,7 +316,7 @@ func main() {
}
if !*remove {
// write maps to database
if err := tpReader.WriteToDatabase(*flush, *verbose, *disable_reverse); err != nil {
if err := tpReader.WriteToDatabase(*flush, *verbose, lCfg.DisableReverse); err != nil {
log.Fatal("Could not write to database: ", err)
}
var dstIds, revDstIDs, rplIds, rpfIds, actIds, aapIDs, shgIds, alsIds, lcrIds, dcsIds, rspIDs, resIDs, aatIDs, ralsIDs []string
@@ -331,7 +415,7 @@ func main() {
}
}
} else {
if err := tpReader.RemoveFromDatabase(*verbose, *disable_reverse); err != nil {
if err := tpReader.RemoveFromDatabase(*verbose, lCfg.DisableReverse); err != nil {
log.Fatal("Could not delete from database: ", err)
}
}

View File

@@ -26,9 +26,9 @@ import (
"github.com/dlintw/goconf"
)
func NewLoaderConfig(cfgPath *string) (lCfg *LoaderCfg, err error) {
func NewLoaderConfig(cfgPath string) (lCfg *LoaderCfg, err error) {
lCfg = NewDefaultLoaderConfig()
c, err := goconf.ReadConfigFile(*cfgPath)
c, err := goconf.ReadConfigFile(cfgPath)
if err != nil {
return nil, errors.New(fmt.Sprintf("Could not open the configuration file: %s", err))
}

View File

@@ -0,0 +1,67 @@
// +build integration
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package config
import (
"reflect"
"testing"
"github.com/cgrates/cgrates/utils"
"github.com/dlintw/goconf"
)
func TestCgrLoaderCfgLoad(t *testing.T) {
cfgPath := "/usr/share/cgrates/conf/samples/cgrloaderconfig/cgr-loader.cfg"
c, err := goconf.ReadConfigFile(cfgPath)
if err != nil {
t.Error(err)
}
rcv := &LoaderCfg{}
if err := rcv.loadConfig(c); err != nil {
t.Error(err)
}
expected := &LoaderCfg{
DataDBType: "redis",
DataDBHost: "127.0.0.1",
DataDBPort: "6379",
DataDBName: "10",
DataDBUser: "cgrates",
DataDBPass: "CGRateS",
StorDBType: "mysql",
StorDBHost: "127.0.0.1",
StorDBPort: "3306",
StorDBName: "cgrates",
StorDBUser: "cgrates",
StorDBPass: "CGRateS",
Tpid: "",
DataPath: "./",
RpcEncoding: "json",
RalsAddress: "127.0.0.1:2012",
CdrstatsAddress: "127.0.0.1:2012",
UsersAddress: "127.0.0.1:2012",
RunId: "",
LoadHistorySize: 10,
Timezone: "Local",
DisableReverse: false,
}
if !reflect.DeepEqual(expected, rcv) {
t.Errorf("Expected: %+v, received: %+v", utils.ToJSON(expected), utils.ToJSON(rcv))
}
}

View File

@@ -22,7 +22,6 @@ import (
"testing"
"github.com/cgrates/cgrates/utils"
"github.com/dlintw/goconf"
)
func TestCgrLoaderCfgSetDefault(t *testing.T) {
@@ -109,42 +108,3 @@ func TestCgrLoaderCfgNewDefault(t *testing.T) {
t.Errorf("Expected: %+v, received: %+v", expected, rcv)
}
}
func TestCgrLoaderCfgLoad(t *testing.T) {
cfgPath := "/usr/share/cgrates/conf/cgrates/cgr-loader.cfg"
c, err := goconf.ReadConfigFile(cfgPath)
if err != nil {
t.Error(err)
}
rcv := &LoaderCfg{}
if err := rcv.loadConfig(c); err != nil {
t.Error(err)
}
expected := &LoaderCfg{
DataDBType: "redis",
DataDBHost: "127.0.0.1",
DataDBPort: "6379",
DataDBName: "10",
DataDBUser: "cgrates",
DataDBPass: "testdatapw",
StorDBType: "mysql",
StorDBHost: "127.0.0.1",
StorDBPort: "3306",
StorDBName: "cgrates",
StorDBUser: "cgrates",
StorDBPass: "teststorpw",
Tpid: "testtpid",
DataPath: "./",
RpcEncoding: "json",
RalsAddress: "testRALsAddress",
CdrstatsAddress: "testcdrstatsaddress",
UsersAddress: "testuseraddress",
RunId: "testrunId",
LoadHistorySize: 10,
Timezone: "Local",
DisableReverse: false,
}
if !reflect.DeepEqual(expected, rcv) {
t.Errorf("Expected: %+v, received: %+v", utils.ToJSON(expected), utils.ToJSON(rcv))
}
}

View File

@@ -1,12 +1,24 @@
# cgr-loader configuration file
[general]
tpid =
dataPath = ./
rpcEncoding = json
ralsAddress = 127.0.0.1:2012
cdrstatsAddress = 127.0.0.1:2012
usersAddress = 127.0.0.1:2012
runId =
loadHistorySize = 10
timezone = Local
disable_reverse = false
[dataDB]
db_type = redis
db_host = 127.0.0.1
db_port = 6379
db_name = 10
db_user = cgrates
db_password = testdatapw
db_password = CGRateS
[storDB]
db_type = mysql
@@ -14,16 +26,4 @@
db_port = 3306
db_name = cgrates
db_user = cgrates
db_password = teststorpw
[general]
tpid = testtpid
dataPath = ./
rpcEncoding = json
ralsAddress = testRALsAddress
cdrstatsAddress = testcdrstatsaddress
usersAddress = testuseraddress
runId = testrunId
loadHistorySize = 10
timezone = Local
disable_reverse = false
db_password = CGRateS

2
glide.lock generated
View File

@@ -123,4 +123,6 @@ imports:
- internal/json
- internal/sasl
- internal/scram
- name: github.com/dlintw/goconf
version: dcc070983490608a14480e3bf943bad464785df5
testImports: []

View File

@@ -41,3 +41,4 @@ import:
- package: github.com/streadway/amqp
- package: github.com/cgrates/radigo
- package: github.com/cgrates/ltcache
- package: github.com/dlintw/goconf