From 509f4a87cda072ef8eefbef41580a3df4861464d Mon Sep 17 00:00:00 2001 From: DanB Date: Tue, 19 Nov 2013 15:44:17 +0100 Subject: [PATCH] Adding local csv loader tests --- engine/loader_test.go | 146 ++++++++++++++++++++++++++++++++++++ engine/storage_interface.go | 6 ++ engine/storage_redis.go | 12 ++- engine/storage_sql_test.go | 48 +++++------- 4 files changed, 175 insertions(+), 37 deletions(-) create mode 100644 engine/loader_test.go diff --git a/engine/loader_test.go b/engine/loader_test.go new file mode 100644 index 000000000..11145f16b --- /dev/null +++ b/engine/loader_test.go @@ -0,0 +1,146 @@ +/* +Rating system designed to be used in VoIP Carriers World +Copyright (C) 2013 ITsysCOM + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +package engine + +import ( + "github.com/cgrates/cgrates/utils" + "github.com/cgrates/cgrates/config" + "testing" + "path" + //"fmt" + "flag" +) + +/* +README: + + Enable local tests by passing '-local' to the go test command + Tests in this file combine end2end tests using both redis and MySQL. + It is expected that the data folder of CGRateS exists at path /usr/share/cgrates/data or passed via command arguments. + Prior running the tests, create database and users by running: + mysql -pyourrootpwd < /usr/share/cgrates/data/storage/mysql/create_db_with_users.sql + What these tests do: + * Connect to redis using 2 handles, one where we store CSV reference data and one where we store data out of storDb, each with it's own db number + * Flush data in each handle to start clean +*/ + +// Globals used +var dataDbCsv, dataDbStor DataStorage // Test data coming from csv files getting inthere +var storDb LoadStorage +var cfg *config.CGRConfig + +// Arguments received via test command +var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args +var dataDir = flag.String("data_dir", "/usr/share/cgrates/data", "CGR data dir path here") +var tpCsvScenario = flag.String("tp_scenario", "prepaid1centpsec", "Use this scenario folder to import tp csv data from") + +// Create connection to dataDb and flush it's data +// Will use 3 different datadbs in order to be able to see differences in data loaded +func TestConnDataDbs(t *testing.T) { + if !*testLocal { + return + } + cfg,_ = config.NewDefaultCGRConfig() + var err error + if dataDbCsv, err = ConfigureDataStorage(cfg.DataDBType, cfg.DataDBHost, cfg.DataDBPort, "13", cfg.DataDBUser, cfg.DataDBPass, cfg.DBDataEncoding); err != nil { + t.Fatal("Error on dataDb connection: ", err.Error()) + } + if dataDbStor, err = ConfigureDataStorage(cfg.DataDBType, cfg.DataDBHost, cfg.DataDBPort, "14", cfg.DataDBUser, cfg.DataDBPass, cfg.DBDataEncoding); err != nil { + t.Fatal("Error on dataDb connection: ", err.Error()) + } +} + +// Create/reset storage tariff plan tables, used as database connectin establishment also +func TestCreateStorTpTables(t *testing.T) { + if !*testLocal { + return + } + var db *MySQLStorage + if d, err := NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass); err != nil { + t.Error("Error on opening database connection: ",err) + return + } else { + db = d.(*MySQLStorage) + storDb = d.(LoadStorage) + } + // Creating the table serves also as reset since there is a drop prior to create + if err := db.CreateTablesFromScript(path.Join(*dataDir, "storage", "mysql", CREATE_TARIFFPLAN_TABLES_SQL)); err != nil { + t.Error("Error on db creation: ", err.Error()) + return // No point in going further + } +} + +// Import data from csv files into dataDbCsv +func TestLoadFromCSV(t *testing.T) { + if !*testLocal { + return + } + var err error + for fn, v := range FileValidators { + if err = ValidateCSVData(path.Join(*dataDir, "tariffplans", *tpCsvScenario, fn), v.Rule); err != nil { + t.Error("Failed validating data: ", err.Error()) + } + } + loader := NewFileCSVReader(dataDbCsv, ',', + path.Join(*dataDir, "tariffplans", *tpCsvScenario, utils.DESTINATIONS_CSV), + path.Join(*dataDir, "tariffplans", *tpCsvScenario, utils.TIMINGS_CSV), + path.Join(*dataDir, "tariffplans", *tpCsvScenario, utils.RATES_CSV), + path.Join(*dataDir, "tariffplans", *tpCsvScenario, utils.DESTINATION_RATES_CSV), + path.Join(*dataDir, "tariffplans", *tpCsvScenario, utils.RATING_PLANS_CSV), + path.Join(*dataDir, "tariffplans", *tpCsvScenario, utils.RATING_PROFILES_CSV), + path.Join(*dataDir, "tariffplans", *tpCsvScenario, utils.ACTIONS_CSV), + path.Join(*dataDir, "tariffplans", *tpCsvScenario, utils.ACTION_TIMINGS_CSV), + path.Join(*dataDir, "tariffplans", *tpCsvScenario, utils.ACTION_TRIGGERS_CSV), + path.Join(*dataDir, "tariffplans", *tpCsvScenario, utils.ACCOUNT_ACTIONS_CSV), + ) + + if err = loader.LoadDestinations(); err != nil { + t.Error("Failed loading destinations: ", err.Error()) + } + if err = loader.LoadTimings(); err != nil { + t.Error("Failed loading timings: ", err.Error()) + } + if err = loader.LoadRates(); err != nil { + t.Error("Failed loading rates: ", err.Error()) + } + if err = loader.LoadDestinationRates(); err != nil { + t.Error("Failed loading destination rates: ", err.Error()) + } + if err = loader.LoadRatingPlans(); err != nil { + t.Error("Failed loading rating plans: ", err.Error()) + } + if err = loader.LoadRatingProfiles(); err != nil { + t.Error("Failed loading rating profiles: ", err.Error()) + } + if err = loader.LoadActions(); err != nil { + t.Error("Failed loading actions: ", err.Error()) + } + if err = loader.LoadActionTimings(); err != nil { + t.Error("Failed loading action timings: ", err.Error()) + } + if err = loader.LoadActionTriggers(); err != nil { + t.Error("Failed loading action triggers: ", err.Error()) + } + if err = loader.LoadAccountActions(); err != nil { + t.Error("Failed loading account actions: ", err.Error()) + } + if err := loader.WriteToDatabase(true, false); err != nil { + t.Error("Could not write data into dataDb: ", err.Error()) + } +} diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 14089bce6..7b99afafe 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -50,6 +50,12 @@ const ( MEDIATOR_SOURCE = "MED" SCHED_SOURCE = "SCH" RATER_SOURCE = "RAT" + // Some consts used in tests + CREATE_CDRS_TABLES_SQL = "create_cdrs_tables.sql" + CREATE_COSTDETAILS_TABLES_SQL = "create_costdetails_tables.sql" + CREATE_MEDIATOR_TABLES_SQL = "create_mediator_tables.sql" + CREATE_TARIFFPLAN_TABLES_SQL = "create_tariffplan_tables.sql" + TEST_SQL = "TEST_SQL" ) type Storage interface { diff --git a/engine/storage_redis.go b/engine/storage_redis.go index df9206088..3bfb77fcd 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -108,14 +108,12 @@ func (rs *RedisStorage) PreCache(dKeys, rpKeys []string) (err error) { } // Used to check if specific subject is stored using prefix key attached to entity -func (rs *RedisStorage) ExistsData(entity, subject string) (bool, error) { - switch entity { - case DESTINATION_PREFIX: - return rs.db.Exists(DESTINATION_PREFIX + subject) - case RATING_PLAN_PREFIX: - return rs.db.Exists(RATING_PLAN_PREFIX + subject) +func (rs *RedisStorage) ExistsData(category, subject string) (bool, error) { + switch category { + case DESTINATION_PREFIX, RATING_PLAN_PREFIX, RATING_PROFILE_PREFIX, ACTION_PREFIX, ACTION_TIMING_PREFIX, USER_BALANCE_PREFIX: + return rs.db.Exists(category + subject) } - return false, errors.New("Unsupported entity in ExistsData") + return false, errors.New("Unsupported category in ExistsData") } func (rs *RedisStorage) GetRatingPlan(key string) (rp *RatingPlan, err error) { diff --git a/engine/storage_sql_test.go b/engine/storage_sql_test.go index b927b5cba..202fc451d 100644 --- a/engine/storage_sql_test.go +++ b/engine/storage_sql_test.go @@ -24,7 +24,6 @@ import ( "testing" "path" "fmt" - "flag" ) /* @@ -37,21 +36,10 @@ README: It is expected that the data folder of CGRateS exists at path /usr/share/cgrates/data. Prior running the tests, create database and users by running: - mysql -pyourrootpwd < /usr/share/cgrates/data/storage/mysql/create_db_with_users.sql + mysql -pyourrootpwd < /usr/share/cgrates/data/storage/mysql/create_mysql_with_users.sql */ -const ( - CREATE_CDRS_TABLES_SQL = "create_cdrs_tables.sql" - CREATE_COSTDETAILS_TABLES_SQL = "create_costdetails_tables.sql" - CREATE_MEDIATOR_TABLES_SQL = "create_mediator_tables.sql" - CREATE_TARIFFPLAN_TABLES_SQL = "create_tariffplan_tables.sql" - TEST_SQL = "TEST_SQL" -) - -var db *MySQLStorage -var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args -var scriptsPath = flag.String("scripts_path", "/usr/share/cgrates/data/storage/mysql", "Overwrite default scripts path here") - +var mysql *MySQLStorage func TestCreateTables(t *testing.T) { if !*testLocal { @@ -62,16 +50,16 @@ func TestCreateTables(t *testing.T) { t.Error("Error on opening database connection: ",err) return } else { - db = d.(*MySQLStorage) + mysql = d.(*MySQLStorage) } for _, scriptName := range []string{CREATE_CDRS_TABLES_SQL, CREATE_COSTDETAILS_TABLES_SQL, CREATE_MEDIATOR_TABLES_SQL, CREATE_TARIFFPLAN_TABLES_SQL} { - if err := db.CreateTablesFromScript(path.Join(*scriptsPath, scriptName)); err != nil { - t.Error("Error on db creation: ", err.Error()) + if err := mysql.CreateTablesFromScript(path.Join(*dataDir, "storage", "mysql", scriptName)); err != nil { + t.Error("Error on mysql creation: ", err.Error()) return // No point in going further } } for _, tbl := range []string{utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_EXTRA} { - if _, err := db.Db.Query(fmt.Sprintf("SELECT 1 from %s", tbl)); err != nil { + if _, err := mysql.Db.Query(fmt.Sprintf("SELECT 1 from %s", tbl)); err != nil { t.Error(err.Error()) } } @@ -83,19 +71,19 @@ func TestRemoveData(t *testing.T) { } // Create Timings tm := &utils.TPTiming{Id:"ALWAYS", StartTime:"00:00:00"} - if err := db.SetTPTiming(TEST_SQL,tm); err != nil { + if err := mysql.SetTPTiming(TEST_SQL,tm); err != nil { t.Error(err.Error()) } - if tmgs, err := db.GetTpTimings(TEST_SQL, tm.Id); err != nil { + if tmgs, err := mysql.GetTpTimings(TEST_SQL, tm.Id); err != nil { t.Error(err.Error()) } else if len(tmgs) == 0 { t.Error("Could not store TPTiming") } // Remove Timings - if err := db.RemTPData(utils.TBL_TP_TIMINGS, TEST_SQL, tm.Id); err != nil { + if err := mysql.RemTPData(utils.TBL_TP_TIMINGS, TEST_SQL, tm.Id); err != nil { t.Error(err.Error()) } - if tmgs, err := db.GetTpTimings(TEST_SQL, tm.Id); err != nil { + if tmgs, err := mysql.GetTpTimings(TEST_SQL, tm.Id); err != nil { t.Error(err.Error()) } else if len(tmgs) != 0 { t.Error("Did not remove TPTiming") @@ -103,19 +91,19 @@ func TestRemoveData(t *testing.T) { // Create RatingProfile ras := []*utils.TPRatingActivation{&utils.TPRatingActivation{ActivationTime: "2012-01-01T00:00:00Z", RatingPlanId:"RETAIL1"}} rp := &utils.TPRatingProfile{TPid:TEST_SQL, LoadId:TEST_SQL, Tenant:"cgrates.org", TOR:"call", Direction:"*out", Subject:"*any", RatingPlanActivations:ras} - if err := db.SetTPRatingProfiles(TEST_SQL, map[string]*utils.TPRatingProfile{rp.KeyId():rp}); err != nil { + if err := mysql.SetTPRatingProfiles(TEST_SQL, map[string]*utils.TPRatingProfile{rp.KeyId():rp}); err != nil { t.Error(err.Error()) } - if rps, err := db.GetTpRatingProfiles(rp); err != nil { + if rps, err := mysql.GetTpRatingProfiles(rp); err != nil { t.Error(err.Error()) } else if len(rps) == 0{ t.Error("Could not store TPRatingProfile") } // Remove RatingProfile - if err := db.RemTPData(utils.TBL_TP_RATE_PROFILES, rp.TPid, rp.LoadId, rp.Tenant, rp.TOR, rp.Direction, rp.Subject); err != nil { + if err := mysql.RemTPData(utils.TBL_TP_RATE_PROFILES, rp.TPid, rp.LoadId, rp.Tenant, rp.TOR, rp.Direction, rp.Subject); err != nil { t.Error(err.Error()) } - if rps, err := db.GetTpRatingProfiles(rp); err != nil { + if rps, err := mysql.GetTpRatingProfiles(rp); err != nil { t.Error(err.Error()) } else if len(rps) != 0{ t.Error("Did not remove TPRatingProfile") @@ -124,19 +112,19 @@ func TestRemoveData(t *testing.T) { // Create AccountActions aa := &utils.TPAccountActions{TPid:TEST_SQL, LoadId:TEST_SQL, Tenant:"cgrates.org", Account:"1001", Direction:"*out", ActionTimingsId:"PREPAID_10", ActionTriggersId:"STANDARD_TRIGGERS"} - if err := db.SetTPAccountActions(aa.TPid, map[string]*utils.TPAccountActions{aa.KeyId(): aa}); err != nil { + if err := mysql.SetTPAccountActions(aa.TPid, map[string]*utils.TPAccountActions{aa.KeyId(): aa}); err != nil { t.Error(err.Error()) } - if aas, err := db.GetTpAccountActions(aa); err != nil { + if aas, err := mysql.GetTpAccountActions(aa); err != nil { t.Error(err.Error()) } else if len(aas) == 0 { t.Error("Could not create TPAccountActions") } // Remove AccountActions - if err := db.RemTPData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.TPid, aa.LoadId, aa.Tenant, aa.Account, aa.Direction); err != nil { + if err := mysql.RemTPData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.TPid, aa.LoadId, aa.Tenant, aa.Account, aa.Direction); err != nil { t.Error(err.Error()) } - if aas, err := db.GetTpAccountActions(aa); err != nil { + if aas, err := mysql.GetTpAccountActions(aa); err != nil { t.Error(err.Error()) } else if len(aas) != 0 { t.Error("Did not remove TPAccountActions")