From f2386c88f3f27372bbf42bf6435f2f5d8d48a38d Mon Sep 17 00:00:00 2001 From: DanB Date: Sun, 23 Aug 2015 15:48:14 +0200 Subject: [PATCH] ApierV2.LoadTariffPlanFromFolder with support for returning loadInstance data, various localtest cache tests fixes --- apier/v1/apier_local_test.go | 17 ++-- apier/v2/apier.go | 136 +++++++++++++++++++++++++++ engine/tp_reader.go | 8 +- general_tests/tutorial_local_test.go | 10 +- 4 files changed, 156 insertions(+), 15 deletions(-) diff --git a/apier/v1/apier_local_test.go b/apier/v1/apier_local_test.go index 0ac5989b0..eabab92a5 100644 --- a/apier/v1/apier_local_test.go +++ b/apier/v1/apier_local_test.go @@ -825,7 +825,7 @@ func TestApierGetCacheStats(t *testing.T) { return } var rcvStats *utils.CacheStats - expectedStats := &utils.CacheStats{Destinations: 3, RatingPlans: 1, RatingProfiles: 2, Actions: 2} + expectedStats := &utils.CacheStats{Destinations: 3, RatingPlans: 1, RatingProfiles: 2, Actions: 2, LastLoadId: utils.NOT_AVAILABLE, LastLoadTime: utils.NOT_AVAILABLE} var args utils.AttrCacheStats if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil { t.Error("Got error on ApierV1.GetCacheStats: ", err.Error()) @@ -1221,7 +1221,7 @@ func TestApierResetDataBeforeLoadFromFolder(t *testing.T) { t.Error("Calling ApierV1.ReloadCache got reply: ", reply) } var rcvStats *utils.CacheStats - expectedStats := &utils.CacheStats{} + expectedStats := &utils.CacheStats{LastLoadId: utils.NOT_AVAILABLE, LastLoadTime: utils.NOT_AVAILABLE} var args utils.AttrCacheStats if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil { t.Error("Got error on ApierV1.GetCacheStats: ", err.Error()) @@ -1267,12 +1267,17 @@ func TestApierResetDataAfterLoadFromFolder(t *testing.T) { t.Error("Calling ApierV1.ReloadCache got reply: ", reply) } var rcvStats *utils.CacheStats - expectedStats := &utils.CacheStats{Destinations: 4, RatingPlans: 3, RatingProfiles: 3, Actions: 5, DerivedChargers: 2} var args utils.AttrCacheStats if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil { t.Error("Got error on ApierV1.GetCacheStats: ", err.Error()) - } else if !reflect.DeepEqual(rcvStats, expectedStats) { - t.Errorf("Calling ApierV1.GetCacheStats received: %v, expected: %v", rcvStats, expectedStats) + } else { + if rcvStats.Destinations != 4 || + rcvStats.RatingPlans != 3 || + rcvStats.RatingProfiles != 3 || + rcvStats.Actions != 5 || + rcvStats.DerivedChargers != 2 { + t.Errorf("Calling ApierV1.GetCacheStats received: %v", rcvStats) + } } } @@ -1751,7 +1756,7 @@ func TestApierGetCacheStats2(t *testing.T) { return } var rcvStats *utils.CacheStats - expectedStats := new(utils.CacheStats) + expectedStats := &utils.CacheStats{LastLoadId: utils.NOT_AVAILABLE, LastLoadTime: utils.NOT_AVAILABLE} var args utils.AttrCacheStats if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil { t.Error("Got error on ApierV1.GetCacheStats: ", err.Error()) diff --git a/apier/v2/apier.go b/apier/v2/apier.go index 21ad66c3e..4d2252e5b 100644 --- a/apier/v2/apier.go +++ b/apier/v2/apier.go @@ -19,6 +19,12 @@ along with this program. If not, see package v2 import ( + "errors" + "fmt" + "os" + "path" + "strings" + "github.com/cgrates/cgrates/apier/v1" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" @@ -120,3 +126,133 @@ func (self *ApierV2) LoadDerivedChargers(attrs AttrLoadDerivedChargers, reply *s *reply = v1.OK return nil } + +func (self *ApierV2) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder, reply *engine.LoadInstance) error { + if len(attrs.FolderPath) == 0 { + return fmt.Errorf("%s:%s", utils.ErrMandatoryIeMissing.Error(), "FolderPath") + } + if fi, err := os.Stat(attrs.FolderPath); err != nil { + if strings.HasSuffix(err.Error(), "no such file or directory") { + return utils.ErrInvalidPath + } + return utils.NewErrServerError(err) + } else if !fi.IsDir() { + return utils.ErrInvalidPath + } + loader := engine.NewTpReader(self.RatingDb, self.AccountDb, engine.NewFileCSVStorage(utils.CSV_SEP, + path.Join(attrs.FolderPath, utils.DESTINATIONS_CSV), + path.Join(attrs.FolderPath, utils.TIMINGS_CSV), + path.Join(attrs.FolderPath, utils.RATES_CSV), + path.Join(attrs.FolderPath, utils.DESTINATION_RATES_CSV), + path.Join(attrs.FolderPath, utils.RATING_PLANS_CSV), + path.Join(attrs.FolderPath, utils.RATING_PROFILES_CSV), + path.Join(attrs.FolderPath, utils.SHARED_GROUPS_CSV), + path.Join(attrs.FolderPath, utils.LCRS_CSV), + path.Join(attrs.FolderPath, utils.ACTIONS_CSV), + path.Join(attrs.FolderPath, utils.ACTION_PLANS_CSV), + path.Join(attrs.FolderPath, utils.ACTION_TRIGGERS_CSV), + path.Join(attrs.FolderPath, utils.ACCOUNT_ACTIONS_CSV), + path.Join(attrs.FolderPath, utils.DERIVED_CHARGERS_CSV), + path.Join(attrs.FolderPath, utils.CDR_STATS_CSV), + path.Join(attrs.FolderPath, utils.USERS_CSV), + path.Join(attrs.FolderPath, utils.ALIASES_CSV), + ), "", self.Config.DefaultTimezone, self.Config.LoadHistorySize) + if err := loader.LoadAll(); err != nil { + return utils.NewErrServerError(err) + } + if attrs.DryRun { + *reply = engine.LoadInstance{LoadId: utils.DRYRUN} + return nil // Mission complete, no errors + } + + if attrs.Validate { + if !loader.IsValid() { + return errors.New("invalid data") + } + } + + if err := loader.WriteToDatabase(attrs.FlushDb, false); err != nil { + return utils.NewErrServerError(err) + } + // Make sure the items are in the cache + dstIds, _ := loader.GetLoadedIds(utils.DESTINATION_PREFIX) + dstKeys := make([]string, len(dstIds)) + for idx, dId := range dstIds { + dstKeys[idx] = utils.DESTINATION_PREFIX + dId // Cache expects them as redis keys + } + rplIds, _ := loader.GetLoadedIds(utils.RATING_PLAN_PREFIX) + rpKeys := make([]string, len(rplIds)) + for idx, rpId := range rplIds { + rpKeys[idx] = utils.RATING_PLAN_PREFIX + rpId + } + rpfIds, _ := loader.GetLoadedIds(utils.RATING_PROFILE_PREFIX) + rpfKeys := make([]string, len(rpfIds)) + for idx, rpfId := range rpfIds { + rpfKeys[idx] = utils.RATING_PROFILE_PREFIX + rpfId + } + actIds, _ := loader.GetLoadedIds(utils.ACTION_PREFIX) + actKeys := make([]string, len(actIds)) + for idx, actId := range actIds { + actKeys[idx] = utils.ACTION_PREFIX + actId + } + shgIds, _ := loader.GetLoadedIds(utils.SHARED_GROUP_PREFIX) + shgKeys := make([]string, len(shgIds)) + for idx, shgId := range shgIds { + shgKeys[idx] = utils.SHARED_GROUP_PREFIX + shgId + } + aliases, _ := loader.GetLoadedIds(utils.ALIASES_PREFIX) + alsKeys := make([]string, len(aliases)) + for idx, alias := range aliases { + alsKeys[idx] = utils.ALIASES_PREFIX + alias + } + lcrIds, _ := loader.GetLoadedIds(utils.LCR_PREFIX) + lcrKeys := make([]string, len(lcrIds)) + for idx, lcrId := range lcrIds { + lcrKeys[idx] = utils.LCR_PREFIX + lcrId + } + dcs, _ := loader.GetLoadedIds(utils.DERIVEDCHARGERS_PREFIX) + dcsKeys := make([]string, len(dcs)) + for idx, dc := range dcs { + dcsKeys[idx] = utils.DERIVEDCHARGERS_PREFIX + dc + } + aps, _ := loader.GetLoadedIds(utils.ACTION_TIMING_PREFIX) + engine.Logger.Info("ApierV1.LoadTariffPlanFromFolder, reloading cache.") + + if err := self.RatingDb.CacheRatingPrefixValues(map[string][]string{ + utils.DESTINATION_PREFIX: dstKeys, + utils.RATING_PLAN_PREFIX: rpKeys, + utils.RATING_PROFILE_PREFIX: rpfKeys, + utils.LCR_PREFIX: lcrKeys, + utils.DERIVEDCHARGERS_PREFIX: dcsKeys, + utils.ACTION_PREFIX: actKeys, + utils.SHARED_GROUP_PREFIX: shgKeys, + }); err != nil { + return err + } + if err := self.AccountDb.CacheAccountingPrefixValues(map[string][]string{ + utils.ALIASES_PREFIX: alsKeys, + }); err != nil { + return err + } + if len(aps) != 0 && self.Sched != nil { + engine.Logger.Info("ApierV1.LoadTariffPlanFromFolder, reloading scheduler.") + self.Sched.LoadActionPlans(self.RatingDb) + self.Sched.Restart() + } + cstKeys, _ := loader.GetLoadedIds(utils.CDR_STATS_PREFIX) + if len(cstKeys) != 0 && self.CdrStatsSrv != nil { + if err := self.CdrStatsSrv.ReloadQueues(cstKeys, nil); err != nil { + return err + } + } + + userKeys, _ := loader.GetLoadedIds(utils.USERS_PREFIX) + if len(userKeys) != 0 && self.Users != nil { + var r string + if err := self.Users.ReloadUsers("", &r); err != nil { + return err + } + } + *reply = *loader.GetLoadInstance() + return nil +} diff --git a/engine/tp_reader.go b/engine/tp_reader.go index 7593de308..f86da841f 100644 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -1328,12 +1328,12 @@ func (tpr *TpReader) WriteToDatabase(flush, verbose bool) (err error) { log.Print("\t", al.GetId()) } } + ldInst := tpr.GetLoadInstance() if verbose { - ldInst := tpr.GetLoadInstance() log.Printf("LoadHistory, instance: %+v\n", ldInst) - if err = tpr.accountingStorage.AddLoadHistory(ldInst, tpr.loadHistSize); err != nil { - return err - } + } + if err = tpr.accountingStorage.AddLoadHistory(ldInst, tpr.loadHistSize); err != nil { + return err } return } diff --git a/general_tests/tutorial_local_test.go b/general_tests/tutorial_local_test.go index 8df970c35..59b014f94 100644 --- a/general_tests/tutorial_local_test.go +++ b/general_tests/tutorial_local_test.go @@ -37,6 +37,7 @@ import ( var tutLocalCfgPath string var tutFsLocalCfg *config.CGRConfig var tutLocalRpc *rpc.Client +var loadInst engine.LoadInstance // Share load information between tests func TestTutLocalInitCfg(t *testing.T) { if !*testLocal { @@ -100,12 +101,11 @@ func TestTutLocalLoadTariffPlanFromFolder(t *testing.T) { if !*testLocal { return } - reply := "" attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tariffplans", "tutorial")} - if err := tutLocalRpc.Call("ApierV1.LoadTariffPlanFromFolder", attrs, &reply); err != nil { + if err := tutLocalRpc.Call("ApierV2.LoadTariffPlanFromFolder", attrs, &loadInst); err != nil { t.Error(err) - } else if reply != "OK" { - t.Error(reply) + } else if loadInst.LoadId == "" { + t.Error("Empty loadId received, loadInstance: ", loadInst) } time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups } @@ -118,7 +118,7 @@ func TestTutLocalCacheStats(t *testing.T) { var rcvStats *utils.CacheStats expectedStats := &utils.CacheStats{Destinations: 4, RatingPlans: 3, RatingProfiles: 8, Actions: 7, SharedGroups: 1, Aliases: 2, - DerivedChargers: 1, LcrProfiles: 5, CdrStats: 6, Users: 3} + DerivedChargers: 1, LcrProfiles: 5, CdrStats: 6, Users: 3, LastLoadId: loadInst.LoadId, LastLoadTime: loadInst.LoadTime.Format(time.RFC3339)} var args utils.AttrCacheStats if err := tutLocalRpc.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil { t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())