Refactored GetTpids and updated cgr-migrator

2026-02-16 13:49:53 +05:00 · 2017-11-15 11:40:03 +00:00
parent 4d093c5511
commit bb691b372a
3 changed files with 61 additions and 89 deletions
--- a/cmd/cgr-migrator/cgr-migrator.go
+++ b/cmd/cgr-migrator/cgr-migrator.go
@@ -33,9 +33,7 @@ import (
 var (
 	sameDBname      = true
 	inDataDB        migrator.MigratorDataDB
-	oldstorDB       engine.Storage
-	oStorDBType     string
-	oDataDBType     string
+	instorDB        engine.Storage
 	oDBDataEncoding string
 	migrate         = flag.String("migrate", "", "Fire up automatic migration *to use multiple values use ',' as separator \n <*set_versions|*cost_details|*accounts|*actions|*action_triggers|*action_plans|*shared_groups> ")
 	version         = flag.Bool("version", false, "Prints the application version.")
@@ -47,12 +45,12 @@ var (
 	outDataDBUser = flag.String("out_datadb_user", config.CgrConfig().DataDbUser, "The DataDb user to sign in as.")
 	outDataDBPass = flag.String("out_datadb_passwd", config.CgrConfig().DataDbPass, "The DataDb user's password.")

-	storDBType = flag.String("stordb_type", config.CgrConfig().StorDBType, "The type of the storDb Database <mysql|postgres>")
-	storDBHost = flag.String("stordb_host", config.CgrConfig().StorDBHost, "The storDb host to connect to.")
-	storDBPort = flag.String("stordb_port", config.CgrConfig().StorDBPort, "The storDb port to bind to.")
-	storDBName = flag.String("stordb_name", config.CgrConfig().StorDBName, "The name/number of the storDb to connect to.")
-	storDBUser = flag.String("stordb_user", config.CgrConfig().StorDBUser, "The storDb user to sign in as.")
-	storDBPass = flag.String("stordb_passwd", config.CgrConfig().StorDBPass, "The storDb user's password.")
+	outStorDBType = flag.String("out_stordb_type", config.CgrConfig().StorDBType, "The type of the storDb Database <mysql|postgres>")
+	outStorDBHost = flag.String("out_stordb_host", config.CgrConfig().StorDBHost, "The storDb host to connect to.")
+	outStorDBPort = flag.String("out_stordb_port", config.CgrConfig().StorDBPort, "The storDb port to bind to.")
+	outStorDBName = flag.String("out_stordb_name", config.CgrConfig().StorDBName, "The name/number of the storDb to connect to.")
+	outStorDBUser = flag.String("out_stordb_user", config.CgrConfig().StorDBUser, "The storDb user to sign in as.")
+	outStorDBPass = flag.String("out_stordb_passwd", config.CgrConfig().StorDBPass, "The storDb user's password.")

 	inDataDBType = flag.String("in_datadb_type", "", "The type of the DataDb Database <redis>")
 	inDataDBHost = flag.String("in_datadb_host", config.CgrConfig().DataDbHost, "The DataDb host to connect to.")
@@ -61,15 +59,15 @@ var (
 	inDataDBUser = flag.String("in_datadb_user", config.CgrConfig().DataDbUser, "The DataDb user to sign in as.")
 	inDataDBPass = flag.String("in_datadb_passwd", config.CgrConfig().DataDbPass, "The DataDb user's password.")

-	oldStorDBType = flag.String("old_stordb_type", "", "The type of the storDb Database <mysql|postgres>")
-	oldStorDBHost = flag.String("old_stordb_host", config.CgrConfig().StorDBHost, "The storDb host to connect to.")
-	oldStorDBPort = flag.String("old_stordb_port", config.CgrConfig().StorDBPort, "The storDb port to bind to.")
-	oldStorDBName = flag.String("old_stordb_name", config.CgrConfig().StorDBName, "The name/number of the storDb to connect to.")
-	oldStorDBUser = flag.String("old_stordb_user", config.CgrConfig().StorDBUser, "The storDb user to sign in as.")
-	oldStorDBPass = flag.String("old_stordb_passwd", config.CgrConfig().StorDBPass, "The storDb user's password.")
+	inStorDBType = flag.String("in_stordb_type", "", "The type of the storDb Database <mysql|postgres>")
+	inStorDBHost = flag.String("in_stordb_host", config.CgrConfig().StorDBHost, "The storDb host to connect to.")
+	inStorDBPort = flag.String("in_stordb_port", config.CgrConfig().StorDBPort, "The storDb port to bind to.")
+	inStorDBName = flag.String("in_stordb_name", config.CgrConfig().StorDBName, "The name/number of the storDb to connect to.")
+	inStorDBUser = flag.String("in_stordb_user", config.CgrConfig().StorDBUser, "The storDb user to sign in as.")
+	inStorDBPass = flag.String("in_stordb_passwd", config.CgrConfig().StorDBPass, "The storDb user's password.")

-	loadHistorySize    = flag.Int("load_history_size", config.CgrConfig().LoadHistorySize, "Limit the number of records in the load history")
-	oldLoadHistorySize = flag.Int("old_load_history_size", 0, "Limit the number of records in the load history")
+	loadHistorySize   = flag.Int("load_history_size", config.CgrConfig().LoadHistorySize, "Limit the number of records in the load history")
+	inLoadHistorySize = flag.Int("in_load_history_size", 0, "Limit the number of records in the load history")

 	dbDataEncoding   = flag.String("dbData_encoding", config.CgrConfig().DBDataEncoding, "The encoding used to store object Data in strings")
 	inDBDataEncoding = flag.String("in_dbData_encoding", "", "The encoding used to store object Data in strings")
@@ -87,11 +85,11 @@ func main() {
 	if migrate != nil && *migrate != "" { // Run migrator
 		if *verbose {
 			log.Print("Initializing DataDB:", *outDataDBType)
-			log.Print("Initializing storDB:", *storDBType)
+			log.Print("Initializing storDB:", *outStorDBType)
 		}
 		var dmOUT *engine.DataManager
 		dmOUT, _ = engine.ConfigureDataStorage(*outDataDBType, *outDataDBHost, *outDataDBPort, *outDataDBName, *outDataDBUser, *outDataDBPass, *dbDataEncoding, config.CgrConfig().CacheConfig, *loadHistorySize)
-		storDB, err := engine.ConfigureStorStorage(*storDBType, *storDBHost, *storDBPort, *storDBName, *storDBUser, *storDBPass, *dbDataEncoding,
+		storDB, err := engine.ConfigureStorStorage(*outStorDBType, *outStorDBHost, *outStorDBPort, *outStorDBName, *outStorDBUser, *outStorDBPass, *dbDataEncoding,
 			config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes)
 		if err != nil {
 			log.Fatal(err)
@@ -113,17 +111,17 @@ func main() {
 		if err != nil {
 			log.Fatal(err)
 		}
-		oldstorDB = storDB
+		instorDB = storDB

 		if *verbose {
-			if *oldStorDBType != "" {
-				log.Print("Initializing oldstorDB:", *oldStorDBType)
+			if *inStorDBType != "" {
+				log.Print("Initializing instorDB:", *inStorDBType)
 			} else {
-				log.Print("Initializing oldstorDB:", *storDBType)
+				log.Print("Initializing instorDB:", *outStorDBType)
 			}
 		}
-		if *oldStorDBType != "" {
-			oldstorDB, err = engine.ConfigureStorStorage(oStorDBType, *oldStorDBHost, *oldStorDBPort, *oldStorDBName, *oldStorDBUser, *oldStorDBPass, *inDBDataEncoding,
+		if *inStorDBType != "" {
+			instorDB, err = engine.ConfigureStorStorage(*inStorDBType, *inStorDBHost, *inStorDBPort, *inStorDBName, *inStorDBUser, *inStorDBPass, *inDBDataEncoding,
 				config.CgrConfig().StorDBMaxOpenConns, config.CgrConfig().StorDBMaxIdleConns, config.CgrConfig().StorDBConnMaxLifetime, config.CgrConfig().StorDBCDRSIndexes)
 			if err != nil {
 				log.Fatal(err)
@@ -135,7 +133,7 @@ func main() {
 		if inDataDBName != outDataDBName {
 			sameDBname = false
 		}
-		m, err := migrator.NewMigrator(dmIN, dmOUT, *outDataDBType, *dbDataEncoding, storDB, *storDBType, inDataDB, *inDataDBType, *inDBDataEncoding, oldstorDB, *oldStorDBType, *dryRun, sameDBname)
+		m, err := migrator.NewMigrator(dmIN, dmOUT, *outDataDBType, *dbDataEncoding, storDB, *inStorDBType, inDataDB, *inDataDBType, *inDBDataEncoding, instorDB, *inStorDBType, *dryRun, sameDBname)
 		if err != nil {
 			log.Fatal(err)
 		}
--- a/engine/storage_mongo_stordb.go
+++ b/engine/storage_mongo_stordb.go
@@ -28,47 +28,35 @@ import (
 	"time"
 )

-func (ms *MongoStorage) GetTpIds(x string) ([]string, error) {
+func (ms *MongoStorage) GetTpIds(colname string) ([]string, error) {
 	tpidMap := make(map[string]bool)
 	session := ms.session.Copy()
 	db := session.DB(ms.db)
 	defer session.Close()
 	var tpids []string
-	if x != "" {
-		if strings.HasPrefix(x, "tp_") {
+	var err error
+	cols := []string{colname}
+	if colname == "" {
+		cols, err = db.CollectionNames()
+		if err != nil {
+			return nil, err
+		}
+	}
+	for _, col := range cols {
+		if strings.HasPrefix(col, "tp_") {
 			tpids := make([]string, 0)
-			if err := db.C(x).Find(nil).Select(bson.M{"tpid": 1}).Distinct("tpid", &tpids); err != nil {
+			if err := db.C(col).Find(nil).Select(bson.M{"tpid": 1}).Distinct("tpid", &tpids); err != nil {
 				return nil, err
 			}
 			for _, tpid := range tpids {
 				tpidMap[tpid] = true
 			}
 		}
-		for tpid := range tpidMap {
-			tpids = append(tpids, tpid)
-		}
-	} else {
-		cols, err := db.CollectionNames()
-		if err != nil {
-			return nil, err
-		}
-		for _, col := range cols {
-			if strings.HasPrefix(col, "tp_") {
-				tpids := make([]string, 0)
-				if err := db.C(col).Find(nil).Select(bson.M{"tpid": 1}).Distinct("tpid", &tpids); err != nil {
-					return nil, err
-				}
-				for _, tpid := range tpids {
-					tpidMap[tpid] = true
-				}
-			}
-		}
-		for tpid := range tpidMap {
-			tpids = append(tpids, tpid)
-		}
+	}
+	for tpid := range tpidMap {
+		tpids = append(tpids, tpid)
 	}
 	return tpids, nil
-
 }

 func (ms *MongoStorage) GetTpTableIds(tpid, table string, distinct utils.TPDistinctIds, filter map[string]string, pag *utils.Paginator) ([]string, error) {
--- a/engine/storage_sql.go
+++ b/engine/storage_sql.go
@@ -120,31 +120,18 @@ func (self *SQLStorage) IsDBEmpty() (resp bool, err error) {

 // update
 // Return a list with all TPids defined in the system, even if incomplete, isolated in some table.
-func (self *SQLStorage) GetTpIds(x string) ([]string, error) {
-	if x != "" {
-		rows, err := self.Db.Query(
-			fmt.Sprintf(" (SELECT tpid FROM %s)", x))
+func (self *SQLStorage) GetTpIds(colname string) ([]string, error) {
+	var rows *sql.Rows
+	var err error
+	if colname != "" {
+		rows, err = self.Db.Query(
+			fmt.Sprintf(" (SELECT tpid FROM %s)", colname))
 		if err != nil {
 			return nil, err
 		}
 		defer rows.Close()
-		ids := make([]string, 0)
-		i := 0
-		for rows.Next() {
-			i++ //Keep here a reference so we know we got at least one
-			var id string
-			err = rows.Scan(&id)
-			if err != nil {
-				return nil, err
-			}
-			ids = append(ids, id)
-		}
-		if i == 0 {
-			return nil, nil
-		}
-		return ids, nil
 	} else {
-		rows, err := self.Db.Query(
+		rows, err = self.Db.Query(
 			fmt.Sprintf(
 				"(SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s) UNION (SELECT tpid FROM %s)",
 				utils.TBLTPTimings,
@@ -171,23 +158,22 @@ func (self *SQLStorage) GetTpIds(x string) ([]string, error) {
 			return nil, err
 		}
 		defer rows.Close()
-		ids := make([]string, 0)
-		i := 0
-		for rows.Next() {
-			i++ //Keep here a reference so we know we got at least one
-			var id string
-			err = rows.Scan(&id)
-			if err != nil {
-				return nil, err
-			}
-			ids = append(ids, id)
-		}
-		if i == 0 {
-			return nil, nil
-		}
-
-		return ids, nil
 	}
+	ids := make([]string, 0)
+	i := 0
+	for rows.Next() {
+		i++ //Keep here a reference so we know we got at least one
+		var id string
+		err = rows.Scan(&id)
+		if err != nil {
+			return nil, err
+		}
+		ids = append(ids, id)
+	}
+	if i == 0 {
+		return nil, nil
+	}
+	return ids, nil
 }

 // ToDo: TEST