diff --git a/apier/v1/apier.go b/apier/v1/apier.go index 8cb2aa023..474962a08 100644 --- a/apier/v1/apier.go +++ b/apier/v1/apier.go @@ -99,7 +99,7 @@ func (apierSv1 *APIerSv1) GetReverseDestination(prefix *string, reply *[]string) return utils.NewErrMandatoryIeMissing("prefix") } var revLst []string - if revLst, err = apierSv1.DataManager.GetReverseDestination(*prefix, false, utils.NonTransactional); err != nil { + if revLst, err = apierSv1.DataManager.GetReverseDestination(*prefix, true, true, utils.NonTransactional); err != nil { return } *reply = revLst @@ -108,7 +108,7 @@ func (apierSv1 *APIerSv1) GetReverseDestination(prefix *string, reply *[]string) // ComputeReverseDestinations will rebuild complete reverse destinations data func (apierSv1 *APIerSv1) ComputeReverseDestinations(ignr *string, reply *string) (err error) { - if err = apierSv1.DataManager.DataDB().RebuildReverseForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil { + if err = apierSv1.DataManager.RebuildReverseForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil { return } *reply = utils.OK @@ -117,7 +117,7 @@ func (apierSv1 *APIerSv1) ComputeReverseDestinations(ignr *string, reply *string // ComputeAccountActionPlans will rebuild complete reverse accountActions data func (apierSv1 *APIerSv1) ComputeAccountActionPlans(ignr *string, reply *string) (err error) { - if err = apierSv1.DataManager.DataDB().RebuildReverseForPrefix(utils.AccountActionPlansPrefix); err != nil { + if err = apierSv1.DataManager.RebuildReverseForPrefix(utils.AccountActionPlansPrefix); err != nil { return } *reply = utils.OK diff --git a/apier/v1/debit_test.go b/apier/v1/debit_test.go index 92a1c682f..92512f747 100644 --- a/apier/v1/debit_test.go +++ b/apier/v1/debit_test.go @@ -80,7 +80,7 @@ func TestDebitUsageWithOptions(t *testing.T) { if err := apierDebitStorage.SetDestinationDrv(dstDe, utils.NonTransactional); err != nil { t.Error(err) } - if err := apierDebitStorage.SetReverseDestinationDrv(dstDe, utils.NonTransactional); err != nil { + if err := apierDebitStorage.SetReverseDestinationDrv(dstDe.Id, dstDe.Prefixes, utils.NonTransactional); err != nil { t.Error(err) } rp1 := &engine.RatingPlan{ diff --git a/apier/v1/replicator.go b/apier/v1/replicator.go index 709521682..b4ce9edfa 100644 --- a/apier/v1/replicator.go +++ b/apier/v1/replicator.go @@ -59,7 +59,7 @@ func (rplSv1 *ReplicatorSv1) GetDestination(key *utils.StringWithOpts, reply *en //GetDestination func (rplSv1 *ReplicatorSv1) GetReverseDestination(key *utils.StringWithOpts, reply *[]string) error { - if rcv, err := rplSv1.dm.DataDB().GetReverseDestinationDrv(key.Arg, true, utils.NonTransactional); err != nil { + if rcv, err := rplSv1.dm.DataDB().GetReverseDestinationDrv(key.Arg, utils.NonTransactional); err != nil { return err } else { *reply = rcv @@ -334,7 +334,7 @@ func (rplSv1 *ReplicatorSv1) SetDestination(dst *engine.DestinationWithOpts, rep // SetReverseDestination func (rplSv1 *ReplicatorSv1) SetReverseDestination(dst *engine.DestinationWithOpts, reply *string) error { - if err := rplSv1.dm.DataDB().SetReverseDestinationDrv(dst.Destination, utils.NonTransactional); err != nil { + if err := rplSv1.dm.DataDB().SetReverseDestinationDrv(dst.Destination.Id, dst.Destination.Prefixes, utils.NonTransactional); err != nil { return err } *reply = utils.OK diff --git a/data/conf/samples/caps_busy/cgrates.json b/data/conf/samples/caps_busy/cgrates.json index 46d138da0..5e5b890a6 100644 --- a/data/conf/samples/caps_busy/cgrates.json +++ b/data/conf/samples/caps_busy/cgrates.json @@ -43,4 +43,4 @@ }, -}, +} diff --git a/engine/account.go b/engine/account.go index 8658ce5ba..94aa025db 100644 --- a/engine/account.go +++ b/engine/account.go @@ -306,7 +306,7 @@ func (acc *Account) getBalancesForPrefix(prefix, category, tor, if len(b.DestinationIDs) > 0 && b.DestinationIDs[utils.ANY] == false { for _, p := range utils.SplitPrefix(prefix, MIN_PREFIX_MATCH) { - if destIDs, err := dm.GetReverseDestination(p, false, utils.NonTransactional); err == nil { + if destIDs, err := dm.GetReverseDestination(p, true, true, utils.NonTransactional); err == nil { foundResult := false allInclude := true // whether it is excluded or included for _, dID := range destIDs { diff --git a/engine/balances.go b/engine/balances.go index defef366b..ef81abec6 100644 --- a/engine/balances.go +++ b/engine/balances.go @@ -185,10 +185,10 @@ func (b *Balance) Clone() *Balance { return n } -func (b *Balance) getMatchingPrefixAndDestID(dest string) (prefix, destId string) { +func (b *Balance) getMatchingPrefixAndDestID(dest string) (prefix, destID string) { if len(b.DestinationIDs) != 0 && b.DestinationIDs[utils.ANY] == false { for _, p := range utils.SplitPrefix(dest, MIN_PREFIX_MATCH) { - if destIDs, err := dm.GetReverseDestination(p, false, utils.NonTransactional); err == nil { + if destIDs, err := dm.GetReverseDestination(p, true, true, utils.NonTransactional); err == nil { for _, dID := range destIDs { if b.DestinationIDs[dID] == true { return p, dID diff --git a/engine/callcost.go b/engine/callcost.go index 2c2d2b3fd..378a7eca5 100644 --- a/engine/callcost.go +++ b/engine/callcost.go @@ -235,7 +235,7 @@ func (cc *CallCost) MatchCCFilter(bf *BalanceFilter) bool { if bf.DestinationIDs != nil && cc.Destination != "" { for _, p := range utils.SplitPrefix(cc.Destination, MIN_PREFIX_MATCH) { if destIDs, err := dm.GetReverseDestination(p, - false, utils.NonTransactional); err == nil { + true, true, utils.NonTransactional); err == nil { for _, dID := range destIDs { if _, ok := (*bf.DestinationIDs)[dID]; ok { foundMatchingDestID = true diff --git a/engine/datamanager.go b/engine/datamanager.go index fa0410455..21bb9e1ee 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -167,7 +167,7 @@ func (dm *DataManager) CacheDataFromDB(prfx string, ids []string, mustBeCached b case utils.DESTINATION_PREFIX: _, err = dm.GetDestination(dataID, false, true, utils.NonTransactional) case utils.REVERSE_DESTINATION_PREFIX: - _, err = dm.GetReverseDestination(dataID, true, utils.NonTransactional) + _, err = dm.GetReverseDestination(dataID, false, true, utils.NonTransactional) case utils.RATING_PLAN_PREFIX: _, err = dm.GetRatingPlan(dataID, true, utils.NonTransactional) case utils.RATING_PROFILE_PREFIX: @@ -305,6 +305,50 @@ func (dm *DataManager) CacheDataFromDB(prfx string, ids []string, mustBeCached b return } +func (dm *DataManager) RebuildReverseForPrefix(prefix string) (err error) { + switch prefix { + case utils.REVERSE_DESTINATION_PREFIX: + if err = dm.dataDB.RemoveKeysForPrefix(prefix); err != nil { + return + } + var keys []string + if keys, err = dm.dataDB.GetKeysForPrefix(utils.DESTINATION_PREFIX); err != nil { + return + } + for _, key := range keys { + var dest *Destination + if dest, err = dm.GetDestination(key[len(utils.DESTINATION_PREFIX):], false, true, utils.NonTransactional); err != nil { + return err + } + if err = dm.SetReverseDestination(dest, utils.NonTransactional); err != nil { + return err + } + } + case utils.AccountActionPlansPrefix: + if err = dm.dataDB.RemoveKeysForPrefix(prefix); err != nil { + return + } + var keys []string + if keys, err = dm.dataDB.GetKeysForPrefix(utils.ACTION_PLAN_PREFIX); err != nil { + return + } + for _, key := range keys { + var apl *ActionPlan + if apl, err = dm.GetActionPlan(key[len(utils.ACTION_PLAN_PREFIX):], true, utils.NonTransactional); err != nil { + return err + } + for acntID := range apl.AccountIDs { + if err = dm.SetAccountActionPlans(acntID, []string{apl.Id}, false); err != nil { + return err + } + } + } + default: + return utils.ErrInvalidKey + } + return +} + func (dm *DataManager) GetDestination(key string, cacheRead, cacheWrite bool, transactionID string) (dest *Destination, err error) { if dm == nil { err = utils.ErrNoDatabaseConn @@ -345,9 +389,9 @@ func (dm *DataManager) GetDestination(key string, cacheRead, cacheWrite bool, tr } } if cacheWrite { - if errCh := Cache.Set(utils.CacheDestinations, key, dest, nil, - cacheCommit(transactionID), transactionID); errCh != nil { - return nil, errCh + if err := Cache.Set(utils.CacheDestinations, key, dest, nil, + cacheCommit(transactionID), transactionID); err != nil { + return nil, err } } return @@ -384,9 +428,30 @@ func (dm *DataManager) RemoveDestination(destID string, transactionID string) (e err = utils.ErrNoDatabaseConn return } + + var oldDst *Destination + if oldDst, err = dm.GetDestination(destID, true, false, + transactionID); err != nil && err != utils.ErrNotFound { + return + } + if err = dm.dataDB.RemoveDestinationDrv(destID, transactionID); err != nil { return } + if err = Cache.Remove(utils.CacheDestinations, destID, + cacheCommit(transactionID), transactionID); err != nil { + return + } + if oldDst == nil { + return utils.ErrNotFound + } + for _, prfx := range oldDst.Prefixes { + if err = dm.dataDB.RemoveReverseDestinationDrv(destID, prfx, transactionID); err != nil { + return + } + dm.GetReverseDestination(prfx, false, true, transactionID) // it will recache the destination + } + if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaDestinations]; itm.Replicate { var reply string dm.connMgr.Call(config.CgrConfig().DataDbCfg().RplConns, nil, utils.ReplicatorSv1RemoveDestination, @@ -406,7 +471,7 @@ func (dm *DataManager) SetReverseDestination(dest *Destination, transactionID st err = utils.ErrNoDatabaseConn return } - if err = dm.dataDB.SetReverseDestinationDrv(dest, transactionID); err != nil { + if err = dm.dataDB.SetReverseDestinationDrv(dest.Id, dest.Prefixes, transactionID); err != nil { return } if config.CgrConfig().DataDbCfg().Items[utils.MetaReverseDestinations].Replicate { @@ -421,12 +486,20 @@ func (dm *DataManager) SetReverseDestination(dest *Destination, transactionID st } func (dm *DataManager) GetReverseDestination(prefix string, - skipCache bool, transactionID string) (ids []string, err error) { + cacheRead, cacheWrite bool, transactionID string) (ids []string, err error) { if dm == nil { err = utils.ErrNoDatabaseConn return } - ids, err = dm.dataDB.GetReverseDestinationDrv(prefix, skipCache, transactionID) + if cacheRead { + if x, ok := Cache.Get(utils.CacheReverseDestinations, prefix); ok { + if x == nil { + return nil, utils.ErrNotFound + } + return x.([]string), nil + } + } + ids, err = dm.dataDB.GetReverseDestinationDrv(prefix, transactionID) if err != nil { if itm := config.CgrConfig().DataDbCfg().Items[utils.MetaReverseDestinations]; err == utils.ErrNotFound && itm.Remote { if err = dm.connMgr.Call(config.CgrConfig().DataDbCfg().RmtConns, nil, @@ -443,22 +516,34 @@ func (dm *DataManager) GetReverseDestination(prefix string, } if err != nil { err = utils.CastRPCErr(err) - return nil, err + if err == utils.ErrNotFound && cacheWrite { + if errCh := Cache.Set(utils.CacheReverseDestinations, prefix, nil, nil, + cacheCommit(transactionID), transactionID); errCh != nil { + return nil, errCh + } + } + return + } + } + if cacheWrite { + if errCh := Cache.Set(utils.CacheReverseDestinations, prefix, ids, nil, + cacheCommit(transactionID), transactionID); errCh != nil { + return nil, errCh } } return } func (dm *DataManager) UpdateReverseDestination(oldDest, newDest *Destination, - transactionID string) error { + transactionID string) (err error) { if dm == nil { return utils.ErrNoDatabaseConn } if oldDest == nil { - return dm.dataDB.UpdateReverseDestinationDrv(newDest.Id, nil, newDest.Prefixes, transactionID) + return dm.dataDB.SetReverseDestinationDrv(newDest.Id, newDest.Prefixes, transactionID) } - var obsoletePrefixes []string + cCommit := cacheCommit(transactionID) var addedPrefixes []string for _, oldPrefix := range oldDest.Prefixes { var found bool @@ -469,7 +554,13 @@ func (dm *DataManager) UpdateReverseDestination(oldDest, newDest *Destination, } } if !found { - obsoletePrefixes = append(obsoletePrefixes, oldPrefix) + if err = dm.dataDB.RemoveReverseDestinationDrv(newDest.Id, oldPrefix, transactionID); err != nil { + return + } + if err = Cache.Remove(utils.CacheReverseDestinations, oldPrefix, + cCommit, transactionID); err != nil { + return + } } } @@ -485,7 +576,7 @@ func (dm *DataManager) UpdateReverseDestination(oldDest, newDest *Destination, addedPrefixes = append(addedPrefixes, newPrefix) } } - return dm.dataDB.UpdateReverseDestinationDrv(newDest.Id, obsoletePrefixes, addedPrefixes, transactionID) + return dm.dataDB.SetReverseDestinationDrv(newDest.Id, addedPrefixes, transactionID) } func (dm *DataManager) GetAccount(id string) (acc *Account, err error) { diff --git a/engine/destinations.go b/engine/destinations.go index 193794474..4b881a692 100644 --- a/engine/destinations.go +++ b/engine/destinations.go @@ -71,7 +71,7 @@ func (d *Destination) AddPrefix(pfx string) { // Reverse search in cache to see if prefix belongs to destination id func CachedDestHasPrefix(destId, prefix string) bool { - if cached, err := dm.GetReverseDestination(prefix, false, utils.NonTransactional); err == nil { + if cached, err := dm.GetReverseDestination(prefix, true, true, utils.NonTransactional); err == nil { return utils.IsSliceMember(cached, destId) } return false diff --git a/engine/destinations_test.go b/engine/destinations_test.go index 2c9c58708..63d2bf259 100644 --- a/engine/destinations_test.go +++ b/engine/destinations_test.go @@ -83,7 +83,7 @@ func TestDestinationGetExists(t *testing.T) { } func TestDestinationReverseGetExistsCache(t *testing.T) { - dm.GetReverseDestination("0256", false, utils.NonTransactional) + dm.GetReverseDestination("0256", true, true, utils.NonTransactional) if _, ok := Cache.Get(utils.CacheReverseDestinations, "0256"); !ok { t.Error("Destination not cached:", err) } diff --git a/engine/model_helpers.go b/engine/model_helpers.go index 160586408..02cc6e6b3 100644 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -1767,7 +1767,6 @@ func ThresholdProfileToAPI(th *ThresholdProfile) (tpTH *utils.TPThresholdProfile } type FilterMdls []*FilterMdl -type TpFilterS []*FilterMdl // CSVHeader return the header for csv fields as a slice of string func (tps FilterMdls) CSVHeader() (result []string) { diff --git a/engine/ratingprofile.go b/engine/ratingprofile.go index 2dcfa40a4..844fe68bd 100644 --- a/engine/ratingprofile.go +++ b/engine/ratingprofile.go @@ -178,7 +178,7 @@ func (rpf *RatingProfile) GetRatingPlansForPrefix(cd *CallDescriptor) (err error } } else { for _, p := range utils.SplitPrefix(cd.Destination, MIN_PREFIX_MATCH) { - if destIDs, err := dm.GetReverseDestination(p, false, utils.NonTransactional); err == nil { + if destIDs, err := dm.GetReverseDestination(p, true, true, utils.NonTransactional); err == nil { var bestWeight *float64 for _, dID := range destIDs { var timeChecker bool diff --git a/engine/storage_interface.go b/engine/storage_interface.go index 87346a596..26052d9b6 100644 --- a/engine/storage_interface.go +++ b/engine/storage_interface.go @@ -34,8 +34,7 @@ type Storage interface { Close() Flush(string) error GetKeysForPrefix(string) ([]string, error) - RebuildReverseForPrefix(string) error - RemoveReverseForPrefix(string) error + RemoveKeysForPrefix(string) error GetVersions(itm string) (vrs Versions, err error) SetVersions(vrs Versions, overwrite bool) (err error) RemoveVersions(vrs Versions) (err error) @@ -57,9 +56,9 @@ type DataDB interface { GetDestinationDrv(string, string) (*Destination, error) SetDestinationDrv(*Destination, string) error RemoveDestinationDrv(string, string) error - SetReverseDestinationDrv(*Destination, string) error - GetReverseDestinationDrv(string, bool, string) ([]string, error) - UpdateReverseDestinationDrv(dstID string, obsoletePrefixes, addedPrefixes []string, transactionID string) error + RemoveReverseDestinationDrv(string, string, string) error + SetReverseDestinationDrv(string, []string, string) error + GetReverseDestinationDrv(string, string) ([]string, error) GetActionsDrv(string) (Actions, error) SetActionsDrv(string, Actions) error RemoveActionsDrv(string) error diff --git a/engine/storage_internal_datadb.go b/engine/storage_internal_datadb.go index 2237f0c5f..f3ff52d5a 100644 --- a/engine/storage_internal_datadb.go +++ b/engine/storage_internal_datadb.go @@ -97,68 +97,14 @@ func (iDB *InternalDB) GetKeysForPrefix(prefix string) (ids []string, err error) return } -// RebuildReverseForPrefix will build the reverse destinations -func (iDB *InternalDB) RebuildReverseForPrefix(prefix string) (err error) { - switch prefix { - case utils.REVERSE_DESTINATION_PREFIX: - var keys []string - if keys, err = iDB.GetKeysForPrefix(prefix); err != nil { - return - } - for _, key := range keys { - Cache.RemoveWithoutReplicate(utils.CacheReverseDestinations, key, - cacheCommit(utils.NonTransactional), utils.NonTransactional) - } - - if keys, err = iDB.GetKeysForPrefix(utils.DESTINATION_PREFIX); err != nil { - return - } - for _, key := range keys { - var dest *Destination - if dest, err = iDB.GetDestinationDrv(key[len(utils.DESTINATION_PREFIX):], - utils.NonTransactional); err != nil { - return - } - if err = iDB.SetReverseDestinationDrv(dest, utils.NonTransactional); err != nil { - return - } - } - case utils.AccountActionPlansPrefix: - default: - return utils.ErrInvalidKey +func (iDB *InternalDB) RemoveKeysForPrefix(prefix string) (err error) { + var keys []string + if keys, err = iDB.GetKeysForPrefix(prefix); err != nil { + return } - return -} - -// RemoveReverseForPrefix implementation -func (iDB *InternalDB) RemoveReverseForPrefix(prefix string) (err error) { - switch prefix { - case utils.REVERSE_DESTINATION_PREFIX: - var keys []string - if keys, err = iDB.GetKeysForPrefix(prefix); err != nil { - return - } - for _, key := range keys { - Cache.RemoveWithoutReplicate(utils.CacheReverseDestinations, key, - cacheCommit(utils.NonTransactional), utils.NonTransactional) - } - keys, err = iDB.GetKeysForPrefix(utils.DESTINATION_PREFIX) - if err != nil { - return err - } - for _, key := range keys { - var dest *Destination - if dest, err = iDB.GetDestinationDrv(key[len(utils.DESTINATION_PREFIX):], - utils.NonTransactional); err != nil { - return - } - if err = iDB.RemoveDestinationDrv(dest.Id, utils.NonTransactional); err != nil { - return - } - } - case utils.AccountActionPlansPrefix: - default: - return utils.ErrInvalidKey + for _, key := range keys { + Cache.RemoveWithoutReplicate(utils.CacheReverseDestinations, key, + cacheCommit(utils.NonTransactional), utils.NonTransactional) } return } @@ -303,23 +249,35 @@ func (iDB *InternalDB) SetDestinationDrv(dest *Destination, transactionID string } func (iDB *InternalDB) RemoveDestinationDrv(destID string, transactionID string) (err error) { - // get destination for prefix list - var d *Destination - if d, err = iDB.GetDestinationDrv(destID, transactionID); err != nil { - return - } Cache.RemoveWithoutReplicate(utils.CacheDestinations, destID, cacheCommit(transactionID), transactionID) - for _, prefix := range d.Prefixes { - Cache.RemoveWithoutReplicate(utils.CacheReverseDestinations, prefix, - cacheCommit(utils.NonTransactional), utils.NonTransactional) + return +} + +func (iDB *InternalDB) RemoveReverseDestinationDrv(dstID, prfx, transactionID string) (err error) { + var revDst []string + if Cache.HasItem(utils.CacheReverseDestinations, prfx) { + x, ok := Cache.Get(utils.CacheReverseDestinations, prfx) + if !ok || x == nil { + return utils.ErrNotFound + } + revDst = x.([]string) + } + mpRevDst := utils.NewStringSet(revDst) + mpRevDst.Remove(dstID) + if mpRevDst.Size() != 0 { + Cache.SetWithoutReplicate(utils.CacheReverseDestinations, prfx, mpRevDst.AsSlice(), nil, + cacheCommit(transactionID), transactionID) + } else { + Cache.RemoveWithoutReplicate(utils.CacheReverseDestinations, prfx, + cacheCommit(transactionID), transactionID) } return } -func (iDB *InternalDB) SetReverseDestinationDrv(dest *Destination, transactionID string) (err error) { +func (iDB *InternalDB) SetReverseDestinationDrv(destID string, prefixes []string, transactionID string) (err error) { var revDst []string - for _, p := range dest.Prefixes { + for _, p := range prefixes { if Cache.HasItem(utils.CacheReverseDestinations, p) { x, ok := Cache.Get(utils.CacheReverseDestinations, p) if !ok || x == nil { @@ -328,7 +286,7 @@ func (iDB *InternalDB) SetReverseDestinationDrv(dest *Destination, transactionID revDst = x.([]string) } mpRevDst := utils.NewStringSet(revDst) - mpRevDst.Add(dest.Id) + mpRevDst.Add(destID) // for ReverseDestination we will use Groups Cache.SetWithoutReplicate(utils.CacheReverseDestinations, p, mpRevDst.AsSlice(), nil, cacheCommit(utils.NonTransactional), utils.NonTransactional) @@ -336,59 +294,15 @@ func (iDB *InternalDB) SetReverseDestinationDrv(dest *Destination, transactionID return } -func (iDB *InternalDB) GetReverseDestinationDrv(prefix string, - skipCache bool, transactionID string) (ids []string, err error) { +func (iDB *InternalDB) GetReverseDestinationDrv(prefix, transactionID string) (ids []string, err error) { if x, ok := Cache.Get(utils.CacheReverseDestinations, prefix); ok && x != nil { - ids := x.([]string) - if len(ids) != 0 { - return ids, nil + if ids = x.([]string); len(ids) != 0 { + return } } return nil, utils.ErrNotFound } -func (iDB *InternalDB) UpdateReverseDestinationDrv(dstID string, - obsoletePrefixes, addedPrefixes []string, transactionID string) (err error) { - var mpRevDst utils.StringSet - // remove id for all obsolete prefixes - cCommit := cacheCommit(transactionID) - for _, obsoletePrefix := range obsoletePrefixes { - if Cache.HasItem(utils.CacheReverseDestinations, obsoletePrefix) { - x, ok := Cache.Get(utils.CacheReverseDestinations, obsoletePrefix) - if !ok || x == nil { - return utils.ErrNotFound - } - mpRevDst = utils.NewStringSet(x.([]string)) - if _, has := mpRevDst[dstID]; has { - delete(mpRevDst, dstID) - } - // for ReverseDestination we will use Groups - Cache.SetWithoutReplicate(utils.CacheReverseDestinations, obsoletePrefix, mpRevDst.AsSlice(), nil, - cacheCommit(utils.NonTransactional), utils.NonTransactional) - } - - Cache.RemoveWithoutReplicate(utils.CacheReverseDestinations, obsoletePrefix, - cCommit, transactionID) - } - // add the id to all new prefixes - for _, addedPrefix := range addedPrefixes { - if Cache.HasItem(utils.CacheReverseDestinations, addedPrefix) { - x, ok := Cache.Get(utils.CacheReverseDestinations, addedPrefix) - if !ok || x == nil { - return utils.ErrNotFound - } - mpRevDst = utils.NewStringSet(x.([]string)) - } else { - mpRevDst = make(utils.StringSet) - } - mpRevDst.Add(dstID) - // for ReverseDestination we will use Groups - Cache.SetWithoutReplicate(utils.CacheReverseDestinations, addedPrefix, mpRevDst.AsSlice(), nil, - cacheCommit(utils.NonTransactional), utils.NonTransactional) - } - return -} - func (iDB *InternalDB) GetActionsDrv(id string) (acts Actions, err error) { if x, ok := Cache.Get(utils.CacheActions, id); ok && x != nil { return x.(Actions), err diff --git a/engine/storage_mongo_datadb.go b/engine/storage_mongo_datadb.go index 49b1f096f..7037e924c 100644 --- a/engine/storage_mongo_datadb.go +++ b/engine/storage_mongo_datadb.go @@ -367,34 +367,6 @@ func (ms *MongoStorage) EnsureIndexes(cols ...string) (err error) { return } -func (ms *MongoStorage) getColNameForPrefix(prefix string) (string, bool) { - res, ok := map[string]string{ - utils.DESTINATION_PREFIX: ColDst, - utils.REVERSE_DESTINATION_PREFIX: ColRds, - utils.ACTION_PREFIX: ColAct, - utils.ACTION_PLAN_PREFIX: ColApl, - utils.AccountActionPlansPrefix: ColAAp, - utils.TASKS_KEY: ColTsk, - utils.ACTION_TRIGGER_PREFIX: ColAtr, - utils.RATING_PLAN_PREFIX: ColRpl, - utils.RATING_PROFILE_PREFIX: ColRpf, - utils.ACCOUNT_PREFIX: ColAcc, - utils.SHARED_GROUP_PREFIX: ColShg, - utils.LOADINST_KEY: ColLht, - utils.VERSION_PREFIX: ColVer, - utils.TimingsPrefix: ColTmg, - utils.ResourcesPrefix: ColRes, - utils.ResourceProfilesPrefix: ColRsP, - utils.ThresholdProfilePrefix: ColTps, - utils.StatQueueProfilePrefix: ColSqp, - utils.ThresholdPrefix: ColThs, - utils.FilterPrefix: ColFlt, - utils.RouteProfilePrefix: ColRts, - utils.AttributeProfilePrefix: ColAttr, - }[prefix] - return res, ok -} - // Close disconects the client func (ms *MongoStorage) Close() { if err := ms.client.Disconnect(ms.ctx); err != nil { @@ -423,103 +395,60 @@ func (ms *MongoStorage) SelectDatabase(dbName string) (err error) { return } -// RebuildReverseForPrefix implementation -func (ms *MongoStorage) RebuildReverseForPrefix(prefix string) (err error) { - if !utils.SliceHasMember([]string{utils.AccountActionPlansPrefix, utils.REVERSE_DESTINATION_PREFIX}, prefix) { - return utils.ErrInvalidKey - } - colName, ok := ms.getColNameForPrefix(prefix) - if !ok { +func (ms *MongoStorage) RemoveKeysForPrefix(prefix string) (err error) { + var colName string + switch prefix { + case utils.DESTINATION_PREFIX: + colName = ColDst + case utils.REVERSE_DESTINATION_PREFIX: + colName = ColRds + case utils.ACTION_PREFIX: + colName = ColAct + case utils.ACTION_PLAN_PREFIX: + colName = ColApl + case utils.AccountActionPlansPrefix: + colName = ColAAp + case utils.TASKS_KEY: + colName = ColTsk + case utils.ACTION_TRIGGER_PREFIX: + colName = ColAtr + case utils.RATING_PLAN_PREFIX: + colName = ColRpl + case utils.RATING_PROFILE_PREFIX: + colName = ColRpf + case utils.ACCOUNT_PREFIX: + colName = ColAcc + case utils.SHARED_GROUP_PREFIX: + colName = ColShg + case utils.LOADINST_KEY: + colName = ColLht + case utils.VERSION_PREFIX: + colName = ColVer + case utils.TimingsPrefix: + colName = ColTmg + case utils.ResourcesPrefix: + colName = ColRes + case utils.ResourceProfilesPrefix: + colName = ColRsP + case utils.ThresholdProfilePrefix: + colName = ColTps + case utils.StatQueueProfilePrefix: + colName = ColSqp + case utils.ThresholdPrefix: + colName = ColThs + case utils.FilterPrefix: + colName = ColFlt + case utils.RouteProfilePrefix: + colName = ColRts + case utils.AttributeProfilePrefix: + colName = ColAttr + default: return utils.ErrInvalidKey } + return ms.query(func(sctx mongo.SessionContext) error { - col := ms.getCol(colName) - if _, err := col.DeleteMany(sctx, bson.M{}); err != nil { - return err - } - var keys []string - switch prefix { - case utils.REVERSE_DESTINATION_PREFIX: - if keys, err = ms.GetKeysForPrefix(utils.DESTINATION_PREFIX); err != nil { - return err - } - for _, key := range keys { - dest, err := ms.GetDestinationDrv(key[len(utils.DESTINATION_PREFIX):], utils.NonTransactional) - if err != nil { - return err - } - if err = ms.SetReverseDestinationDrv(dest, utils.NonTransactional); err != nil { - return err - } - } - case utils.AccountActionPlansPrefix: - if keys, err = ms.GetKeysForPrefix(utils.ACTION_PLAN_PREFIX); err != nil { - return err - } - for _, key := range keys { - apl, err := ms.GetActionPlanDrv(key[len(utils.ACTION_PLAN_PREFIX):], true, utils.NonTransactional) - if err != nil { - return err - } - for acntID := range apl.AccountIDs { - if err = ms.SetAccountActionPlansDrv(acntID, []string{apl.Id}, false); err != nil { - return err - } - } - } - } - return nil - }) -} - -// RemoveReverseForPrefix implementation -func (ms *MongoStorage) RemoveReverseForPrefix(prefix string) (err error) { - if !utils.SliceHasMember([]string{utils.AccountActionPlansPrefix, utils.REVERSE_DESTINATION_PREFIX}, prefix) { - return utils.ErrInvalidKey - } - colName, ok := ms.getColNameForPrefix(prefix) - if !ok { - return utils.ErrInvalidKey - } - return ms.query(func(sctx mongo.SessionContext) error { - col := ms.getCol(colName) - - if _, err := col.DeleteMany(sctx, bson.M{}); err != nil { - return err - } - - var keys []string - switch prefix { - case utils.REVERSE_DESTINATION_PREFIX: - if keys, err = ms.GetKeysForPrefix(utils.DESTINATION_PREFIX); err != nil { - return err - } - for _, key := range keys { - dest, err := ms.GetDestinationDrv(key[len(utils.DESTINATION_PREFIX):], utils.NonTransactional) - if err != nil { - return err - } - if err := ms.RemoveDestinationDrv(dest.Id, utils.NonTransactional); err != nil { - return err - } - } - case utils.AccountActionPlansPrefix: - if keys, err = ms.GetKeysForPrefix(utils.ACTION_PLAN_PREFIX); err != nil { - return err - } - for _, key := range keys { - apl, err := ms.GetActionPlanDrv(key[len(utils.ACTION_PLAN_PREFIX):], true, utils.NonTransactional) - if err != nil { - return err - } - for acntID := range apl.AccountIDs { - if err = ms.RemAccountActionPlansDrv(acntID, []string{apl.Id}); err != nil { - return err - } - } - } - } - return nil + _, err := ms.getCol(colName).DeleteMany(sctx, bson.M{}) + return err }) } @@ -705,7 +634,7 @@ func (ms *MongoStorage) GetKeysForPrefix(prefix string) (result []string, err er case utils.FilterIndexPrfx: result, err = ms.getField3(sctx, ColIndx, utils.FilterIndexPrfx, "key") default: - err = fmt.Errorf("unsupported prefix in GetKeysForPrefix: %s", prefix) + err = fmt.Errorf("unsupported prefix in GetKeysForPrefix: %q", prefix) } return err }) @@ -920,52 +849,24 @@ func (ms *MongoStorage) SetDestinationDrv(dest *Destination, transactionID strin func (ms *MongoStorage) RemoveDestinationDrv(destID string, transactionID string) (err error) { - // get destination for prefix list - d, err := ms.GetDestinationDrv(destID, transactionID) - if err != nil { - fmt.Println("get", destID) - return - } - if err = ms.query(func(sctx mongo.SessionContext) (err error) { + return ms.query(func(sctx mongo.SessionContext) (err error) { dr, err := ms.getCol(ColDst).DeleteOne(sctx, bson.M{"key": destID}) if dr.DeletedCount == 0 { - fmt.Println("deletecount", destID) return utils.ErrNotFound } return err - }); err != nil { - return err - } - if errCh := Cache.Remove(utils.CacheDestinations, destID, - cacheCommit(transactionID), transactionID); errCh != nil { - fmt.Println("cache", destID) - return errCh - } - - for _, prefix := range d.Prefixes { - if err = ms.query(func(sctx mongo.SessionContext) (err error) { - _, err = ms.getCol(ColRds).UpdateOne(sctx, bson.M{"key": prefix}, - bson.M{"$pull": bson.M{"value": destID}}) - return err - }); err != nil { - fmt.Println("reverse", destID, "p", prefix) - return err - } - ms.GetReverseDestinationDrv(prefix, true, transactionID) // it will recache the destination - } - return + }) } -func (ms *MongoStorage) GetReverseDestinationDrv(prefix string, skipCache bool, - transactionID string) (ids []string, err error) { - if !skipCache { - if x, ok := Cache.Get(utils.CacheReverseDestinations, prefix); ok { - if x == nil { - return nil, utils.ErrNotFound - } - return x.([]string), nil - } - } +func (ms *MongoStorage) RemoveReverseDestinationDrv(dstID, prfx, transactionID string) (err error) { + return ms.query(func(sctx mongo.SessionContext) (err error) { + _, err = ms.getCol(ColRds).UpdateOne(sctx, bson.M{"key": prfx}, + bson.M{"$pull": bson.M{"value": dstID}}) + return err + }) +} + +func (ms *MongoStorage) GetReverseDestinationDrv(prefix, transactionID string) (ids []string, err error) { var result struct { Key string Value []string @@ -974,10 +875,6 @@ func (ms *MongoStorage) GetReverseDestinationDrv(prefix string, skipCache bool, cur := ms.getCol(ColRds).FindOne(sctx, bson.M{"key": prefix}) if err := cur.Decode(&result); err != nil { if err == mongo.ErrNoDocuments { - if errCh := Cache.Set(utils.CacheReverseDestinations, prefix, nil, nil, - cacheCommit(transactionID), transactionID); errCh != nil { - return errCh - } return utils.ErrNotFound } return err @@ -987,52 +884,14 @@ func (ms *MongoStorage) GetReverseDestinationDrv(prefix string, skipCache bool, return nil, err } ids = result.Value - if errCh := Cache.Set(utils.CacheReverseDestinations, prefix, ids, nil, - cacheCommit(transactionID), transactionID); errCh != nil { - return nil, errCh - } return } -func (ms *MongoStorage) SetReverseDestinationDrv(dest *Destination, - transactionID string) (err error) { - for _, p := range dest.Prefixes { +func (ms *MongoStorage) SetReverseDestinationDrv(destID string, prefixes []string, transactionID string) (err error) { + for _, p := range prefixes { if err = ms.query(func(sctx mongo.SessionContext) (err error) { _, err = ms.getCol(ColRds).UpdateOne(sctx, bson.M{"key": p}, - bson.M{"$addToSet": bson.M{"value": dest.Id}}, - options.Update().SetUpsert(true), - ) - return err - }); err != nil { - return err - } - } - return nil -} - -func (ms *MongoStorage) UpdateReverseDestinationDrv(dstID string, - obsoletePrefixes, addedPrefixes []string, transactionID string) (err error) { - // remove id for all obsolete prefixes - cCommit := cacheCommit(transactionID) - for _, obsoletePrefix := range obsoletePrefixes { - if err = ms.query(func(sctx mongo.SessionContext) (err error) { - _, err = ms.getCol(ColRds).UpdateOne(sctx, bson.M{"key": obsoletePrefix}, - bson.M{"$pull": bson.M{"value": dstID}}) - return err - }); err != nil { - return err - } - if errCh := Cache.Remove(utils.CacheReverseDestinations, obsoletePrefix, - cCommit, transactionID); errCh != nil { - return errCh - } - } - - // add the id to all new prefixes - for _, addedPrefix := range addedPrefixes { - if err = ms.query(func(sctx mongo.SessionContext) (err error) { - _, err = ms.getCol(ColRds).UpdateOne(sctx, bson.M{"key": addedPrefix}, - bson.M{"$addToSet": bson.M{"value": dstID}}, + bson.M{"$addToSet": bson.M{"value": destID}}, options.Update().SetUpsert(true), ) return err diff --git a/engine/storage_redis.go b/engine/storage_redis.go index ef6d7b553..05c58cfba 100644 --- a/engine/storage_redis.go +++ b/engine/storage_redis.go @@ -199,13 +199,9 @@ func (rs *RedisStorage) IsDBEmpty() (resp bool, err error) { return true, nil } -func (rs *RedisStorage) RebuildReverseForPrefix(prefix string) (err error) { - if !utils.SliceHasMember([]string{utils.AccountActionPlansPrefix, utils.REVERSE_DESTINATION_PREFIX}, prefix) { - return utils.ErrInvalidKey - } +func (rs *RedisStorage) RemoveKeysForPrefix(prefix string) (err error) { var keys []string - keys, err = rs.GetKeysForPrefix(prefix) - if err != nil { + if keys, err = rs.GetKeysForPrefix(prefix); err != nil { return } for _, key := range keys { @@ -213,84 +209,7 @@ func (rs *RedisStorage) RebuildReverseForPrefix(prefix string) (err error) { return } } - switch prefix { - case utils.REVERSE_DESTINATION_PREFIX: - if keys, err = rs.GetKeysForPrefix(utils.DESTINATION_PREFIX); err != nil { - return - } - for _, key := range keys { - dest, err := rs.GetDestinationDrv(key[len(utils.DESTINATION_PREFIX):], utils.NonTransactional) - if err != nil { - return err - } - if err = rs.SetReverseDestinationDrv(dest, utils.NonTransactional); err != nil { - return err - } - } - case utils.AccountActionPlansPrefix: - if keys, err = rs.GetKeysForPrefix(utils.ACTION_PLAN_PREFIX); err != nil { - return - } - for _, key := range keys { - apl, err := rs.GetActionPlanDrv(key[len(utils.ACTION_PLAN_PREFIX):], true, utils.NonTransactional) // skipCache on get since loader checks and caches empty data for loaded objects - if err != nil { - return err - } - for acntID := range apl.AccountIDs { - if err = rs.SetAccountActionPlansDrv(acntID, []string{apl.Id}, false); err != nil { - return err - } - } - } - } - return nil -} - -func (rs *RedisStorage) RemoveReverseForPrefix(prefix string) (err error) { - if !utils.SliceHasMember([]string{utils.AccountActionPlansPrefix, utils.REVERSE_DESTINATION_PREFIX}, prefix) { - return utils.ErrInvalidKey - } - var keys []string - keys, err = rs.GetKeysForPrefix(prefix) - if err != nil { - return - } - for _, key := range keys { - if err = rs.Cmd(nil, redis_DEL, key); err != nil { - return - } - } - switch prefix { - case utils.REVERSE_DESTINATION_PREFIX: - if keys, err = rs.GetKeysForPrefix(utils.DESTINATION_PREFIX); err != nil { - return - } - for _, key := range keys { - dest, err := rs.GetDestinationDrv(key[len(utils.DESTINATION_PREFIX):], utils.NonTransactional) - if err != nil { - return err - } - if err := rs.RemoveDestinationDrv(dest.Id, utils.NonTransactional); err != nil { - return err - } - } - case utils.AccountActionPlansPrefix: - if keys, err = rs.GetKeysForPrefix(utils.ACTION_PLAN_PREFIX); err != nil { - return - } - for _, key := range keys { - apl, err := rs.GetActionPlanDrv(key[len(utils.ACTION_PLAN_PREFIX):], true, utils.NonTransactional) // skipCache on get since loader checks and caches empty data for loaded objects - if err != nil { - return err - } - for acntID := range apl.AccountIDs { - if err = rs.RemAccountActionPlansDrv(acntID, []string{apl.Id}); err != nil { - return err - } - } - } - } - return nil + return } func (rs *RedisStorage) getKeysForFilterIndexesKeys(fkeys []string) (keys []string, err error) { @@ -476,35 +395,19 @@ func (rs *RedisStorage) SetDestinationDrv(dest *Destination, transactionID strin return } -func (rs *RedisStorage) GetReverseDestinationDrv(key string, - skipCache bool, transactionID string) (ids []string, err error) { - if !skipCache { - if x, ok := Cache.Get(utils.CacheReverseDestinations, key); ok { - if x == nil { - return nil, utils.ErrNotFound - } - return x.([]string), nil - } - } +func (rs *RedisStorage) GetReverseDestinationDrv(key, transactionID string) (ids []string, err error) { if err = rs.Cmd(&ids, redis_SMEMBERS, utils.REVERSE_DESTINATION_PREFIX+key); err != nil { return } if len(ids) == 0 { - if err = Cache.Set(utils.CacheReverseDestinations, key, nil, nil, - cacheCommit(transactionID), transactionID); err != nil { - return - } err = utils.ErrNotFound - return } - err = Cache.Set(utils.CacheReverseDestinations, key, ids, nil, - cacheCommit(transactionID), transactionID) return } -func (rs *RedisStorage) SetReverseDestinationDrv(dest *Destination, transactionID string) (err error) { - for _, p := range dest.Prefixes { - if err = rs.Cmd(nil, redis_SADD, utils.REVERSE_DESTINATION_PREFIX+p, dest.Id); err != nil { +func (rs *RedisStorage) SetReverseDestinationDrv(destID string, prefixes []string, transactionID string) (err error) { + for _, p := range prefixes { + if err = rs.Cmd(nil, redis_SADD, utils.REVERSE_DESTINATION_PREFIX+p, destID); err != nil { return } } @@ -512,52 +415,11 @@ func (rs *RedisStorage) SetReverseDestinationDrv(dest *Destination, transactionI } func (rs *RedisStorage) RemoveDestinationDrv(destID, transactionID string) (err error) { - // get destination for prefix list - var d *Destination - if d, err = rs.GetDestinationDrv(destID, transactionID); err != nil { - return - } - if err = rs.Cmd(nil, redis_DEL, utils.DESTINATION_PREFIX+destID); err != nil { - return - } - if err = Cache.Remove(utils.CacheDestinations, destID, - cacheCommit(transactionID), transactionID); err != nil { - return - } - if d == nil { - return utils.ErrNotFound - } - for _, prefix := range d.Prefixes { - if err = rs.Cmd(nil, redis_SREM, utils.REVERSE_DESTINATION_PREFIX+prefix, destID); err != nil { - return - } - rs.GetReverseDestinationDrv(prefix, true, transactionID) // it will recache the destination - } - return + return rs.Cmd(nil, redis_DEL, utils.DESTINATION_PREFIX+destID) } -func (rs *RedisStorage) UpdateReverseDestinationDrv(dstID string, - obsoletePrefixes, addedPrefixes []string, transactionID string) (err error) { - // remove id for all obsolete prefixes - cCommit := cacheCommit(transactionID) - for _, obsoletePrefix := range obsoletePrefixes { - if err = rs.Cmd(nil, redis_SREM, - utils.REVERSE_DESTINATION_PREFIX+obsoletePrefix, dstID); err != nil { - return - } - if err = Cache.Remove(utils.CacheReverseDestinations, obsoletePrefix, - cCommit, transactionID); err != nil { - return - } - } - - // add the id to all new prefixes - for _, addedPrefix := range addedPrefixes { - if err = rs.Cmd(nil, redis_SADD, utils.REVERSE_DESTINATION_PREFIX+addedPrefix, dstID); err != nil { - return - } - } - return +func (rs *RedisStorage) RemoveReverseDestinationDrv(dstID, prfx, transactionID string) (err error) { + return rs.Cmd(nil, redis_SREM, utils.REVERSE_DESTINATION_PREFIX+prfx, dstID) } func (rs *RedisStorage) GetActionsDrv(key string) (as Actions, err error) { diff --git a/engine/storage_sql.go b/engine/storage_sql.go index 0c70968f7..3133ca4b7 100644 --- a/engine/storage_sql.go +++ b/engine/storage_sql.go @@ -75,7 +75,7 @@ func (self *SQLStorage) GetKeysForPrefix(prefix string) ([]string, error) { return nil, utils.ErrNotImplemented } -func (self *SQLStorage) RebuildReverseForPrefix(prefix string) error { +func (SQLStorage) RemoveKeysForPrefix(string) error { return utils.ErrNotImplemented } diff --git a/engine/tpreader.go b/engine/tpreader.go index b0183bf3d..45496e54f 100644 --- a/engine/tpreader.go +++ b/engine/tpreader.go @@ -1411,7 +1411,6 @@ func (tpr *TpReader) WriteToDatabase(verbose, disableReverse bool) (err error) { log.Print("Destinations:") } for _, d := range tpr.destinations { - fmt.Println(d.Id) if err = tpr.setDestination(d, disableReverse, utils.NonTransactional); err != nil { return } @@ -1860,7 +1859,7 @@ func (tpr *TpReader) WriteToDatabase(verbose, disableReverse bool) (err error) { if verbose { log.Print("Rebuilding account action plans") } - if err = tpr.dm.DataDB().RebuildReverseForPrefix(utils.AccountActionPlansPrefix); err != nil { + if err = tpr.dm.RebuildReverseForPrefix(utils.AccountActionPlansPrefix); err != nil { return } } @@ -2117,14 +2116,12 @@ func (tpr *TpReader) RemoveFromDatabase(verbose, disableReverse bool) (err error loadIDs := make(map[string]int64) for _, d := range tpr.destinations { if err = tpr.dm.RemoveDestination(d.Id, utils.NonTransactional); err != nil { - fmt.Println("delete", d.Id) return } if verbose { log.Print("\t", d.Id, " : ", d.Prefixes) } } - fmt.Println(4) for _, rp := range tpr.ratingPlans { if err = tpr.dm.RemoveRatingPlan(rp.Id, utils.NonTransactional); err != nil { return @@ -2375,8 +2372,7 @@ func (tpr *TpReader) RemoveFromDatabase(verbose, disableReverse bool) (err error if verbose { log.Print("Removing reverse destinations") } - if err = tpr.dm.DataDB().RemoveReverseForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil { - fmt.Println(1) + if err = tpr.dm.DataDB().RemoveKeysForPrefix(utils.REVERSE_DESTINATION_PREFIX); err != nil { return } } @@ -2384,12 +2380,11 @@ func (tpr *TpReader) RemoveFromDatabase(verbose, disableReverse bool) (err error if verbose { log.Print("Removing account action plans") } - if err = tpr.dm.DataDB().RemoveReverseForPrefix(utils.AccountActionPlansPrefix); err != nil { + if err = tpr.dm.DataDB().RemoveKeysForPrefix(utils.AccountActionPlansPrefix); err != nil { return } } } - fmt.Println(2) //We remove the filters at the end because of indexes if verbose { log.Print("Filters:") @@ -2740,12 +2735,15 @@ func (tpr *TpReader) setDestination(dest *Destination, disableReverse bool, tran return tpr.dm.SetDestination(dest, transID) } var oldDest *Destination - if oldDest, err = tpr.dm.GetDestination(dest.Id, true, false, transID); err != nil && + if oldDest, err = tpr.dm.GetDestination(dest.Id, false, false, transID); err != nil && err != utils.ErrNotFound { return } if err = tpr.dm.SetDestination(dest, transID); err != nil { - fmt.Println(10) + return + } + if err = Cache.Set(utils.CacheDestinations, dest.Id, dest, nil, + cacheCommit(transID), transID); err != nil { return } return tpr.dm.UpdateReverseDestination(oldDest, dest, transID) diff --git a/engine/z_loader_it_test.go b/engine/z_loader_it_test.go index 245dbc8f3..0933f5856 100644 --- a/engine/z_loader_it_test.go +++ b/engine/z_loader_it_test.go @@ -21,7 +21,6 @@ package engine import ( "flag" - "fmt" "path" "reflect" "testing" @@ -206,15 +205,12 @@ func testLoaderITRemoveLoad(t *testing.T) { if err = loader.LoadDispatcherHosts(); err != nil { t.Error("Failed loading Dispatcher hosts: ", err.Error()) } - fmt.Println(utils.ToJSON(loader.destinations)) if err := loader.WriteToDatabase(false, false); err != nil { t.Error("Could not write data into dataDb: ", err.Error()) } - fmt.Println(utils.ToJSON(loader.destinations)) if err := loader.RemoveFromDatabase(false, true); err != nil { t.Error("Could not remove data from dataDb: ", err.Error()) } - fmt.Println(utils.ToJSON(loader.destinations)) } // Loads data from csv files in tp scenario to dataDbCsv diff --git a/engine/z_onstor_it_test.go b/engine/z_onstor_it_test.go index 0343f2487..27cdd579a 100644 --- a/engine/z_onstor_it_test.go +++ b/engine/z_onstor_it_test.go @@ -645,14 +645,14 @@ func testOnStorITCRUDDestinations(t *testing.T) { func testOnStorITCRUDReverseDestinations(t *testing.T) { dst := &Destination{Id: "CRUDReverseDestination", Prefixes: []string{"+494", "+495", "+496"}} dst2 := &Destination{Id: "CRUDReverseDestination", Prefixes: []string{"+497", "+498", "+499"}} - if _, rcvErr := onStor.GetReverseDestination(dst.Id, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { + if _, rcvErr := onStor.GetReverseDestination(dst.Id, false, true, utils.NonTransactional); rcvErr != utils.ErrNotFound { t.Error(rcvErr) } if err := onStor.SetReverseDestination(dst, utils.NonTransactional); err != nil { t.Error(err) } for i := range dst.Prefixes { - if rcv, err := onStor.GetReverseDestination(dst.Prefixes[i], true, utils.NonTransactional); err != nil { + if rcv, err := onStor.GetReverseDestination(dst.Prefixes[i], false, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual([]string{dst.Id}, rcv) { t.Errorf("Expecting: %v, received: %v", []string{dst.Id}, rcv) @@ -662,7 +662,7 @@ func testOnStorITCRUDReverseDestinations(t *testing.T) { t.Error(err) } for i := range dst.Prefixes { - if rcv, err := onStor.GetReverseDestination(dst2.Prefixes[i], true, utils.NonTransactional); err != nil { + if rcv, err := onStor.GetReverseDestination(dst2.Prefixes[i], false, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual([]string{dst2.Id}, rcv) { t.Errorf("Expecting: %v, received: %v", []string{dst.Id}, rcv) @@ -677,7 +677,7 @@ func testOnStorITCRUDReverseDestinations(t *testing.T) { // } // for i := range dst.Prefixes { - if rcv, err := onStor.GetReverseDestination(dst2.Prefixes[i], false, utils.NonTransactional); err != nil { + if rcv, err := onStor.GetReverseDestination(dst2.Prefixes[i], true, true, utils.NonTransactional); err != nil { t.Error(err) } else if !reflect.DeepEqual([]string{dst2.Id}, rcv) { t.Errorf("Expecting: %v, received: %v", []string{dst.Id}, rcv) diff --git a/loaders/loader.go b/loaders/loader.go index 3c557f2a2..5db3f8e73 100644 --- a/loaders/loader.go +++ b/loaders/loader.go @@ -972,7 +972,6 @@ func (ldr *Loader) removeLoadedData(loaderType string, lds map[string][]LoaderDa } func (ldr *Loader) serve(stopChan chan struct{}) (err error) { - fmt.Println(ldr.runDelay) switch ldr.runDelay { case time.Duration(0): // 0 disables the automatic read, maybe done per API return @@ -1001,7 +1000,6 @@ func (ldr *Loader) handleFolder(stopChan chan struct{}) { } } func (ldr *Loader) processFile(_, itmID string) (err error) { - fmt.Println(itmID) loaderType := ldr.getLdrType(itmID) if len(loaderType) == 0 { return diff --git a/migrator/destinations.go b/migrator/destinations.go index f1b66af61..b3b99d410 100644 --- a/migrator/destinations.go +++ b/migrator/destinations.go @@ -101,7 +101,7 @@ func (m *Migrator) migrateCurrentReverseDestinations() (err error) { } for _, id := range ids { id := strings.TrimPrefix(id, utils.REVERSE_DESTINATION_PREFIX) - rdst, err := m.dmIN.DataManager().GetReverseDestination(id, true, utils.NonTransactional) + rdst, err := m.dmIN.DataManager().GetReverseDestination(id, false, true, utils.NonTransactional) if err != nil { return err } diff --git a/utils/file.go b/utils/file.go index 9af418f7c..0c56f1835 100644 --- a/utils/file.go +++ b/utils/file.go @@ -45,7 +45,6 @@ func WatchDir(dirPath string, f func(itmPath, itmID string) error, Logger.Info(fmt.Sprintf("<%s> stop watching path <%s>", sysID, dirPath)) return case ev := <-watcher.Events: - fmt.Println("event", ev.Name) if ev.Op&fsnotify.Create == fsnotify.Create { go func() { //Enable async processing here so we can simultaneously process files if err = f(filepath.Dir(ev.Name), filepath.Base(ev.Name)); err != nil {