Renamed lower service names, renamed LoaderConfig to LoaderSConfig, added configuration file and test for cgr-loader

cmd/cgr-console/cgr-console.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"log"
 	"os"
+	"sort"
 	"strings"
 	"time"
 
@@ -49,9 +50,19 @@ func executeCommand(command string) {
 	}
 	if strings.TrimSpace(command) == "help" {
 		commands := console.GetCommands()
+		orderedKeys := make([]string, len(commands))
 		fmt.Println("Commands:")
-		for name, cmd := range commands {
-			fmt.Print(name, cmd.Usage())
+		for name, _ := range commands {
+			if name != "" {
+				orderedKeys = append(orderedKeys, name)
+			}
+		}
+		sort.Strings(orderedKeys)
+		for _, name := range orderedKeys {
+			if commands[name] == nil {
+				continue
+			}
+			fmt.Println(name, commands[name].Usage())
 		}
 		return
 	}

config/cgrloadercfg.go

@@ -0,0 +1,191 @@
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/cgrates/cgrates/utils"
+	"github.com/dlintw/goconf"
+)
+
+func NewLoaderConfig(cfgPath *string) (lCfg *LoaderCfg, err error) {
+	lCfg = NewDefaultLoaderConfig()
+	c, err := goconf.ReadConfigFile(*cfgPath)
+	if err != nil {
+		return nil, errors.New(fmt.Sprintf("Could not open the configuration file: %s", err))
+	}
+	if err = lCfg.loadConfig(c); err != nil {
+		return lCfg, err
+	}
+	return
+}
+
+func NewDefaultLoaderConfig() (lCfg *LoaderCfg) {
+	lCfg = new(LoaderCfg)
+	lCfg.setDefaults()
+	return lCfg
+}
+
+type LoaderCfg struct {
+	DataDBType      string
+	DataDBHost      string
+	DataDBPort      string
+	DataDBName      string
+	DataDBUser      string
+	DataDBPass      string
+	DataDBEncoding  string
+	StorDBType      string
+	StorDBHost      string
+	StorDBPort      string
+	StorDBName      string
+	StorDBUser      string
+	StorDBPass      string
+	Flush           bool
+	Tpid            string
+	DataPath        string
+	Version         bool
+	Verbose         bool
+	DryRun          bool
+	Validate        bool
+	Stats           bool
+	FromStorDB      bool
+	ToStorDB        bool
+	RpcEncoding     string
+	RalsAddress     string
+	CdrstatsAddress string
+	UsersAddress    string
+	RunId           string
+	LoadHistorySize int
+	Timezone        string
+	DisableReverse  bool
+	FlushStorDB     bool
+	Remove          bool
+}
+
+func (lCfg *LoaderCfg) setDefaults() {
+	lCfg.DataDBType = cgrCfg.DataDbType
+	lCfg.DataDBHost = utils.MetaDynamic
+	lCfg.DataDBPort = utils.MetaDynamic
+	lCfg.DataDBName = utils.MetaDynamic
+	lCfg.DataDBUser = utils.MetaDynamic
+	lCfg.DataDBPass = utils.MetaDynamic
+	lCfg.StorDBType = cgrCfg.StorDBType
+	lCfg.StorDBHost = utils.MetaDynamic
+	lCfg.StorDBPort = utils.MetaDynamic
+	lCfg.StorDBName = utils.MetaDynamic
+	lCfg.StorDBUser = utils.MetaDynamic
+	lCfg.StorDBPass = utils.MetaDynamic
+	lCfg.Flush = false
+	lCfg.Tpid = ""
+	lCfg.DataPath = "./"
+	lCfg.Version = false
+	lCfg.Verbose = false
+	lCfg.DryRun = false
+	lCfg.Validate = false
+	lCfg.Stats = false
+	lCfg.FromStorDB = false
+	lCfg.ToStorDB = false
+	lCfg.RpcEncoding = "json"
+	lCfg.RalsAddress = cgrCfg.RPCJSONListen
+	lCfg.CdrstatsAddress = cgrCfg.RPCJSONListen
+	lCfg.UsersAddress = cgrCfg.RPCJSONListen
+	lCfg.RunId = ""
+	lCfg.LoadHistorySize = cgrCfg.LoadHistorySize
+	lCfg.Timezone = cgrCfg.DefaultTimezone
+	lCfg.DisableReverse = false
+	lCfg.FlushStorDB = false
+	lCfg.Remove = false
+}
+
+func (ldrCfg *LoaderCfg) loadConfig(c *goconf.ConfigFile) (err error) {
+	var hasOpt bool
+	//dataDB
+	if hasOpt = c.HasOption("dataDB", "db_type"); hasOpt {
+		ldrCfg.DataDBType, err = c.GetString("dataDB", "db_type")
+	}
+	if hasOpt = c.HasOption("dataDB", "db_host"); hasOpt {
+		ldrCfg.DataDBHost, err = c.GetString("dataDB", "db_host")
+	}
+	if hasOpt = c.HasOption("dataDB", "db_port"); hasOpt {
+		ldrCfg.DataDBPort, err = c.GetString("dataDB", "db_port")
+	}
+	if hasOpt = c.HasOption("dataDB", "db_name"); hasOpt {
+		ldrCfg.DataDBName, err = c.GetString("dataDB", "db_name")
+	}
+	if hasOpt = c.HasOption("dataDB", "db_user"); hasOpt {
+		ldrCfg.DataDBUser, err = c.GetString("dataDB", "db_user")
+	}
+	if hasOpt = c.HasOption("dataDB", "db_password"); hasOpt {
+		ldrCfg.DataDBPass, err = c.GetString("dataDB", "db_password")
+	}
+	//storDB
+	if hasOpt = c.HasOption("storDB", "db_type"); hasOpt {
+		ldrCfg.StorDBType, err = c.GetString("storDB", "db_type")
+	}
+	if hasOpt = c.HasOption("storDB", "db_host"); hasOpt {
+		ldrCfg.StorDBHost, err = c.GetString("storDB", "db_host")
+	}
+	if hasOpt = c.HasOption("storDB", "db_port"); hasOpt {
+		ldrCfg.StorDBPort, err = c.GetString("storDB", "db_port")
+	}
+	if hasOpt = c.HasOption("storDB", "db_name"); hasOpt {
+		ldrCfg.StorDBName, err = c.GetString("storDB", "db_name")
+	}
+	if hasOpt = c.HasOption("storDB", "db_user"); hasOpt {
+		ldrCfg.StorDBUser, err = c.GetString("storDB", "db_user")
+	}
+	if hasOpt = c.HasOption("storDB", "db_password"); hasOpt {
+		ldrCfg.StorDBPass, err = c.GetString("storDB", "db_password")
+	}
+	//general
+	if hasOpt = c.HasOption("general", "tpid"); hasOpt {
+		ldrCfg.Tpid, err = c.GetString("general", "tpid")
+	}
+	if hasOpt = c.HasOption("general", "dataPath"); hasOpt {
+		ldrCfg.DataPath, err = c.GetString("general", "dataPath")
+	}
+	if hasOpt = c.HasOption("general", "rpcEncoding"); hasOpt {
+		ldrCfg.RpcEncoding, err = c.GetString("general", "rpcEncoding")
+	}
+	if hasOpt = c.HasOption("general", "ralsAddress"); hasOpt {
+		ldrCfg.RalsAddress, err = c.GetString("general", "ralsAddress")
+	}
+	if hasOpt = c.HasOption("general", "cdrstatsAddress"); hasOpt {
+		ldrCfg.CdrstatsAddress, err = c.GetString("general", "cdrstatsAddress")
+	}
+	if hasOpt = c.HasOption("general", "usersAddress"); hasOpt {
+		ldrCfg.UsersAddress, err = c.GetString("general", "usersAddress")
+	}
+	if hasOpt = c.HasOption("general", "runId"); hasOpt {
+		ldrCfg.RunId, err = c.GetString("general", "runId")
+	}
+	if hasOpt = c.HasOption("general", "loadHistorySize"); hasOpt {
+		ldrCfg.LoadHistorySize, err = c.GetInt("general", "loadHistorySize")
+	}
+	if hasOpt = c.HasOption("general", "timezone"); hasOpt {
+		ldrCfg.Timezone, err = c.GetString("general", "timezone")
+	}
+	if hasOpt = c.HasOption("general", "disable_reverse"); hasOpt {
+		ldrCfg.DisableReverse, err = c.GetBool("general", "disable_reverse")
+	}
+
+	return err
+}

config/cgrloadercfg_test.go

@@ -0,0 +1,150 @@
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+package config
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/cgrates/cgrates/utils"
+	"github.com/dlintw/goconf"
+)
+
+func TestCgrLoaderCfgSetDefault(t *testing.T) {
+	rcv := &LoaderCfg{}
+	rcv.setDefaults()
+	expected := &LoaderCfg{
+		DataDBType:      cgrCfg.DataDbType,
+		DataDBHost:      utils.MetaDynamic,
+		DataDBPort:      utils.MetaDynamic,
+		DataDBName:      utils.MetaDynamic,
+		DataDBUser:      utils.MetaDynamic,
+		DataDBPass:      utils.MetaDynamic,
+		StorDBType:      cgrCfg.StorDBType,
+		StorDBHost:      utils.MetaDynamic,
+		StorDBPort:      utils.MetaDynamic,
+		StorDBName:      utils.MetaDynamic,
+		StorDBUser:      utils.MetaDynamic,
+		StorDBPass:      utils.MetaDynamic,
+		Flush:           false,
+		Tpid:            "",
+		DataPath:        "./",
+		Version:         false,
+		Verbose:         false,
+		DryRun:          false,
+		Validate:        false,
+		Stats:           false,
+		FromStorDB:      false,
+		ToStorDB:        false,
+		RpcEncoding:     "json",
+		RalsAddress:     cgrCfg.RPCJSONListen,
+		CdrstatsAddress: cgrCfg.RPCJSONListen,
+		UsersAddress:    cgrCfg.RPCJSONListen,
+		RunId:           "",
+		LoadHistorySize: cgrCfg.LoadHistorySize,
+		Timezone:        cgrCfg.DefaultTimezone,
+		DisableReverse:  false,
+		FlushStorDB:     false,
+		Remove:          false,
+	}
+
+	if !reflect.DeepEqual(expected, rcv) {
+		t.Errorf("Expected: %+v, received: %+v", expected, rcv)
+	}
+}
+
+func TestCgrLoaderCfgNewDefault(t *testing.T) {
+	rcv := NewDefaultLoaderConfig()
+	expected := &LoaderCfg{
+		DataDBType:      cgrCfg.DataDbType,
+		DataDBHost:      utils.MetaDynamic,
+		DataDBPort:      utils.MetaDynamic,
+		DataDBName:      utils.MetaDynamic,
+		DataDBUser:      utils.MetaDynamic,
+		DataDBPass:      utils.MetaDynamic,
+		StorDBType:      cgrCfg.StorDBType,
+		StorDBHost:      utils.MetaDynamic,
+		StorDBPort:      utils.MetaDynamic,
+		StorDBName:      utils.MetaDynamic,
+		StorDBUser:      utils.MetaDynamic,
+		StorDBPass:      utils.MetaDynamic,
+		Flush:           false,
+		Tpid:            "",
+		DataPath:        "./",
+		Version:         false,
+		Verbose:         false,
+		DryRun:          false,
+		Validate:        false,
+		Stats:           false,
+		FromStorDB:      false,
+		ToStorDB:        false,
+		RpcEncoding:     "json",
+		RalsAddress:     cgrCfg.RPCJSONListen,
+		CdrstatsAddress: cgrCfg.RPCJSONListen,
+		UsersAddress:    cgrCfg.RPCJSONListen,
+		RunId:           "",
+		LoadHistorySize: cgrCfg.LoadHistorySize,
+		Timezone:        cgrCfg.DefaultTimezone,
+		DisableReverse:  false,
+		FlushStorDB:     false,
+		Remove:          false,
+	}
+
+	if !reflect.DeepEqual(expected, rcv) {
+		t.Errorf("Expected: %+v, received: %+v", expected, rcv)
+	}
+}
+
+func TestCgrLoaderCfgLoad(t *testing.T) {
+	cfgPath := "/usr/share/cgrates/conf/cgrates/cgr-loader.cfg"
+	c, err := goconf.ReadConfigFile(cfgPath)
+	if err != nil {
+		t.Error(err)
+	}
+	rcv := &LoaderCfg{}
+	if err := rcv.loadConfig(c); err != nil {
+		t.Error(err)
+	}
+	expected := &LoaderCfg{
+		DataDBType:      "redis",
+		DataDBHost:      "127.0.0.1",
+		DataDBPort:      "6379",
+		DataDBName:      "10",
+		DataDBUser:      "cgrates",
+		DataDBPass:      "testdatapw",
+		StorDBType:      "mysql",
+		StorDBHost:      "127.0.0.1",
+		StorDBPort:      "3306",
+		StorDBName:      "cgrates",
+		StorDBUser:      "cgrates",
+		StorDBPass:      "teststorpw",
+		Tpid:            "testtpid",
+		DataPath:        "./",
+		RpcEncoding:     "json",
+		RalsAddress:     "testRALsAddress",
+		CdrstatsAddress: "testcdrstatsaddress",
+		UsersAddress:    "testuseraddress",
+		RunId:           "testrunId",
+		LoadHistorySize: 10,
+		Timezone:        "Local",
+		DisableReverse:  false,
+	}
+	if !reflect.DeepEqual(expected, rcv) {
+		t.Errorf("Expected: %+v, received: %+v", utils.ToJSON(expected), utils.ToJSON(rcv))
+	}
+}

config/config.go

@@ -48,7 +48,7 @@ var (
 	dfltKamConnConfig        *KamConnConfig // Default Kamailio Connection configuration
 	dfltHaPoolConfig         *HaPoolConfig
 	dfltAstConnCfg           *AsteriskConnCfg
-	dfltLoaderConfig         *LoaderConfig
+	dfltLoaderConfig         *LoaderSConfig
 	dfltLoaderDataTypeConfig *LoaderDataType
 )
 
@@ -349,7 +349,7 @@ type CGRConfig struct {
 	statsCfg                 *StatSCfg                // Configuration for StatS
 	thresholdSCfg            *ThresholdSCfg           // configuration for ThresholdS
 	supplierSCfg             *SupplierSCfg            // configuration for SupplierS
-	loaderCfg                []*LoaderConfig          // configuration for Loader
+	loaderCfg                []*LoaderSConfig         // configuration for Loader
 	dispatcherSCfg           *DispatcherSCfg          // configuration for Dispatcher
 	MailerServer             string                   // The server to use when sending emails out
 	MailerAuthUser           string                   // Authenticate to email server using this user
@@ -1183,9 +1183,9 @@ func (self *CGRConfig) loadFromJsonCfg(jsnCfg *CgrJsonCfg) (err error) {
 	}
 
 	if jsnLoaderCfg != nil {
-		self.loaderCfg = make([]*LoaderConfig, len(jsnLoaderCfg))
+		self.loaderCfg = make([]*LoaderSConfig, len(jsnLoaderCfg))
 		for idx, profile := range jsnLoaderCfg {
-			self.loaderCfg[idx] = NewDfltLoaderConfig()
+			self.loaderCfg[idx] = NewDfltLoaderSConfig()
 			self.loaderCfg[idx].loadFromJsonCfg(profile)
 		}
 	}
@@ -1436,7 +1436,7 @@ func (cfg *CGRConfig) CacheCfg() CacheConfig {
 	return cfg.cacheConfig
 }
 
-func (cfg *CGRConfig) LoaderCfg() []*LoaderConfig {
+func (cfg *CGRConfig) LoaderCfg() []*LoaderSConfig {
 	return cfg.loaderCfg
 }
 

config/config_test.go

@@ -915,8 +915,8 @@ func TestDbDefaults(t *testing.T) {
 }
 
 func TestCgrCfgLoaderDefaults(t *testing.T) {
-	eCfg := []*LoaderConfig{
-		&LoaderConfig{
+	eCfg := []*LoaderSConfig{
+		&LoaderSConfig{
 			Id:           utils.META_DEFAULT,
 			Enabled:      false,
 			DryRun:       false,

config/loadersconfig.go

@@ -22,15 +22,15 @@ import (
 	"time"
 )
 
-func NewDfltLoaderConfig() *LoaderConfig {
+func NewDfltLoaderSConfig() *LoaderSConfig {
 	if dfltLoaderConfig == nil {
-		return new(LoaderConfig)
+		return new(LoaderSConfig)
 	}
 	dfltVal := *dfltLoaderConfig
 	return &dfltVal
 }
 
-type LoaderConfig struct { // rename to LoaderConfig
+type LoaderSConfig struct { // rename to LoaderSConfig
 	Id             string
 	Enabled        bool
 	DryRun         bool
@@ -78,7 +78,7 @@ func (self *LoaderDataType) loadFromJsonCfg(jsnCfg *LoaderJsonDataType) error {
 	return nil
 }
 
-func (self *LoaderConfig) loadFromJsonCfg(jsnCfg *LoaderJsonCfg) error {
+func (self *LoaderSConfig) loadFromJsonCfg(jsnCfg *LoaderJsonCfg) error {
 	if jsnCfg == nil {
 		return nil
 	}
@@ -127,8 +127,8 @@ func (self *LoaderConfig) loadFromJsonCfg(jsnCfg *LoaderJsonCfg) error {
 }
 
 // Clone itself into a new LoadersConfig
-func (self *LoaderConfig) Clone() *LoaderConfig {
-	clnLoader := new(LoaderConfig)
+func (self *LoaderSConfig) Clone() *LoaderSConfig {
+	clnLoader := new(LoaderSConfig)
 	clnLoader.Id = self.Id
 	clnLoader.Enabled = self.Enabled
 	clnLoader.DryRun = self.DryRun

console/ping.go

@@ -49,21 +49,21 @@ func (self *CmdApierPing) Name() string {
 
 func (self *CmdApierPing) RpcMethod() string {
 	switch strings.ToLower(self.rpcParams.Item) {
-	case utils.LowerSuppliers:
+	case utils.SuppliersLow:
 		return utils.SupplierSv1Ping
-	case utils.LowerAttributes:
+	case utils.AttributesLow:
 		return utils.AttributeSv1Ping
-	case utils.LowerResources:
+	case utils.ResourcesLow:
 		return utils.ResourceSv1Ping
-	case utils.LowerStatService:
+	case utils.StatServiceLow:
 		return utils.StatSv1Ping
-	case utils.LowerThresholds:
+	case utils.ThresholdsLow:
 		return utils.ThresholdSv1Ping
-	case utils.LowerSessions:
+	case utils.SessionsLow:
 		return utils.SessionSv1Ping
-	case utils.LowerLoader:
+	case utils.LoaderLow:
 		return utils.LoaderSv1Ping
-	case utils.LowerDispatchers:
+	case utils.DispatchersLow:
 		return utils.DispatcherSv1Ping
 	default:
 	}

data/conf/cgrates/cgr-loader.cfg

@@ -0,0 +1,29 @@
+# cgr-loader configuration file
+
+[dataDB]
+	db_type = redis
+	db_host = 127.0.0.1
+	db_port = 6379
+	db_name = 10
+	db_user = cgrates
+	db_password = testdatapw
+
+[storDB]
+	db_type = mysql
+	db_host = 127.0.0.1
+	db_port = 3306
+	db_name = cgrates
+	db_user = cgrates
+	db_password = teststorpw
+
+[general]
+	tpid = testtpid
+	dataPath = ./
+	rpcEncoding = json
+	ralsAddress = testRALsAddress
+	cdrstatsAddress = testcdrstatsaddress
+	usersAddress = testuseraddress
+	runId = testrunId
+	loadHistorySize = 10
+	timezone = Local
+	disable_reverse = false

loaders/loader.go

@@ -38,7 +38,7 @@ type openedCSVFile struct {
 	csvRdr   *csv.Reader
 }
 
-func NewLoader(dm *engine.DataManager, cfg *config.LoaderConfig,
+func NewLoader(dm *engine.DataManager, cfg *config.LoaderSConfig,
 	timezone string) (ldr *Loader) {
 	ldr = &Loader{
 		enabled:       cfg.Enabled,

loaders/loaders.go

@@ -27,7 +27,7 @@ import (
 	"github.com/cgrates/cgrates/utils"
 )
 
-func NewLoaderService(dm *engine.DataManager, ldrsCfg []*config.LoaderConfig,
+func NewLoaderService(dm *engine.DataManager, ldrsCfg []*config.LoaderSConfig,
 	timezone string) (ldrS *LoaderService) {
 	ldrS = &LoaderService{ldrs: make(map[string]*Loader)}
 	for _, ldrCfg := range ldrsCfg {

utils/consts.go

@@ -565,14 +565,14 @@ const (
 
 // Lower service names
 const (
-	LowerSessions    = "sessions"
-	LowerAttributes  = "attributes"
-	LowerSuppliers   = "suppliers"
-	LowerResources   = "resources"
-	LowerStatService = "stats"
-	LowerThresholds  = "thresholds"
-	LowerDispatchers = "dispatchers"
-	LowerLoader      = "loader"
+	SessionsLow    = "sessions"
+	AttributesLow  = "attributes"
+	SuppliersLow   = "suppliers"
+	ResourcesLow   = "resources"
+	StatServiceLow = "stats"
+	ThresholdsLow  = "thresholds"
+	DispatchersLow = "dispatchers"
+	LoaderLow      = "loader"
 )
 
 //Migrator Metas