Update Caching system for Resource,Stat,Threshold,Supplier and Attribute

apier/v1/attributes.go

@@ -59,8 +59,8 @@ type ArgRemoveAttrPrf struct {
 }
 
 //RemAttributeProfile remove a specific Attribute Profile
-func (apierV1 *ApierV1) RemAttributeProfile(arg ArgRemoveAttrPrf, reply *string) error {
-	if missing := utils.MissingStructFields(&arg, []string{"Tenant", "ID", "Contexts"}); len(missing) != 0 { //Params missing
+func (apierV1 *ApierV1) RemAttributeProfile(arg *ArgRemoveAttrPrf, reply *string) error {
+	if missing := utils.MissingStructFields(arg, []string{"Tenant", "ID", "Contexts"}); len(missing) != 0 { //Params missing
 		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	if err := apierV1.DataManager.RemoveAttributeProfile(arg.Tenant, arg.ID, arg.Contexts, utils.NonTransactional, true); err != nil {

apier/v1/attributes_it_test.go

@@ -320,7 +320,8 @@ func testAttributeSUpdateAlsPrf(t *testing.T) {
 		t.Error("Unexpected reply returned", result)
 	}
 	var reply *engine.ExternalAttributeProfile
-	if err := attrSRPC.Call("ApierV1.GetAttributeProfile", &utils.TenantID{Tenant: "cgrates.org", ID: "ApierTest"}, &reply); err != nil {
+	if err := attrSRPC.Call("ApierV1.GetAttributeProfile",
+		&utils.TenantID{Tenant: "cgrates.org", ID: "ApierTest"}, &reply); err != nil {
 		t.Error(err)
 	} else if !reflect.DeepEqual(alsPrf.FilterIDs, reply.FilterIDs) {
 		t.Errorf("Expecting : %+v, received: %+v", alsPrf.FilterIDs, reply.FilterIDs)
@@ -335,13 +336,15 @@ func testAttributeSUpdateAlsPrf(t *testing.T) {
 
 func testAttributeSRemAlsPrf(t *testing.T) {
 	var resp string
-	if err := attrSRPC.Call("ApierV1.RemAttributeProfile", &ArgRemoveAttrPrf{Tenant: alsPrf.Tenant, ID: alsPrf.ID, Contexts: alsPrf.Contexts}, &resp); err != nil {
+	if err := attrSRPC.Call("ApierV1.RemAttributeProfile",
+		&ArgRemoveAttrPrf{Tenant: alsPrf.Tenant, ID: alsPrf.ID, Contexts: alsPrf.Contexts}, &resp); err != nil {
 		t.Error(err)
 	} else if resp != utils.OK {
 		t.Error("Unexpected reply returned", resp)
 	}
 	var reply *engine.ExternalAttributeProfile
-	if err := attrSRPC.Call("ApierV1.GetAttributeProfile", &utils.TenantID{Tenant: "cgrates.org", ID: "ApierTest"}, &reply); err == nil || err.Error() != utils.ErrNotFound.Error() {
+	if err := attrSRPC.Call("ApierV1.GetAttributeProfile",
+		&utils.TenantID{Tenant: "cgrates.org", ID: "ApierTest"}, &reply); err == nil || err.Error() != utils.ErrNotFound.Error() {
 		t.Error(err)
 	}
 }

apier/v1/filter_indexes_it_test.go

@@ -519,7 +519,6 @@ func testV1FIdxSetStatQueueProfileIndexes(t *testing.T) {
 		nil); err != utils.ErrNotFound {
 		t.Error(err)
 	}
-
 }
 
 func testV1FIdxComputeStatQueueProfileIndexes(t *testing.T) {
@@ -672,6 +671,7 @@ func testV1FIdxSecondComputeStatQueueProfileIndexes(t *testing.T) {
 		t.Errorf("Expecting: %+v, received: %+v", expectedRevIDX, utils.ToJSON(indexes))
 	}
 }
+
 func testV1FIdxRemoveStatQueueProfile(t *testing.T) {
 	var resp string
 	tenant := "cgrates.org"
@@ -789,7 +789,6 @@ func testV1FIdxSetResourceProfileIndexes(t *testing.T) {
 		nil); err != utils.ErrNotFound {
 		t.Error(err)
 	}
-
 }
 
 func testV1FIdxComputeResourceProfileIndexes(t *testing.T) {
@@ -1052,7 +1051,6 @@ func testV1FIdxSetSupplierProfileIndexes(t *testing.T) {
 		nil); err != utils.ErrNotFound {
 		t.Error(err)
 	}
-
 }
 
 func testV1FIdxComputeSupplierProfileIndexes(t *testing.T) {

apier/v1/filters.go

@@ -40,7 +40,7 @@ func (self *ApierV1) GetFilter(arg utils.TenantID, reply *engine.Filter) error {
 	if missing := utils.MissingStructFields(&arg, []string{"Tenant", "ID"}); len(missing) != 0 { //Params missing
 		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if fltr, err := self.DataManager.GetFilter(arg.Tenant, arg.ID, true, utils.NonTransactional); err != nil {
+	if fltr, err := self.DataManager.GetFilter(arg.Tenant, arg.ID, false, utils.NonTransactional); err != nil {
 		if err.Error() != utils.ErrNotFound.Error() {
 			err = utils.NewErrServerError(err)
 		}

engine/datamanager.go

@@ -277,7 +277,13 @@ func (dm *DataManager) SetStatQueue(sq *StatQueue) (err error) {
 	if err != nil {
 		return err
 	}
-	return dm.dataDB.SetStoredStatQueueDrv(ssq)
+	if err = dm.dataDB.SetStoredStatQueueDrv(ssq); err != nil {
+		return
+	}
+	if err = dm.CacheDataFromDB(utils.StatQueuePrefix, []string{sq.TenantID()}, true); err != nil {
+		return
+	}
+	return
 }
 
 // RemStatQueue removes the StoredStatQueue and clears the cache for StatQueue
@@ -312,7 +318,13 @@ func (dm *DataManager) GetFilter(tenant, id string, skipCache bool, transactionI
 }
 
 func (dm *DataManager) SetFilter(fltr *Filter) (err error) {
-	return dm.DataDB().SetFilterDrv(fltr)
+	if err = dm.DataDB().SetFilterDrv(fltr); err != nil {
+		return
+	}
+	if err = dm.CacheDataFromDB(utils.FilterPrefix, []string{fltr.TenantID()}, true); err != nil {
+		return
+	}
+	return
 }
 
 func (dm *DataManager) RemoveFilter(tenant, id, transactionID string) (err error) {
@@ -346,7 +358,13 @@ func (dm *DataManager) GetThreshold(tenant, id string, skipCache bool, transacti
 }
 
 func (dm *DataManager) SetThreshold(th *Threshold) (err error) {
-	return dm.DataDB().SetThresholdDrv(th)
+	if err = dm.DataDB().SetThresholdDrv(th); err != nil {
+		return
+	}
+	if err = dm.CacheDataFromDB(utils.ThresholdPrefix, []string{th.TenantID()}, true); err != nil {
+		return
+	}
+	return
 }
 
 func (dm *DataManager) RemoveThreshold(tenant, id, transactionID string) (err error) {
@@ -383,7 +401,9 @@ func (dm *DataManager) SetThresholdProfile(th *ThresholdProfile, withIndex bool)
 	if err = dm.DataDB().SetThresholdProfileDrv(th); err != nil {
 		return err
 	}
-	cache.RemKey(utils.ThresholdProfilePrefix+utils.ConcatenatedKey(th.Tenant, th.ID), true, "") // ToDo: Remove here with autoreload
+	if err = dm.CacheDataFromDB(utils.ThresholdProfilePrefix, []string{th.TenantID()}, true); err != nil {
+		return
+	}
 	if withIndex {
 		//remove old ThresholdProfile indexes
 		indexerRemove := NewReqFilterIndexer(dm, utils.ThresholdProfilePrefix, th.Tenant)
@@ -457,8 +477,9 @@ func (dm *DataManager) SetStatQueueProfile(sqp *StatQueueProfile, withIndex bool
 	if err = dm.DataDB().SetStatQueueProfileDrv(sqp); err != nil {
 		return err
 	}
-	cache.RemKey(utils.StatQueueProfilePrefix+utils.ConcatenatedKey(sqp.Tenant, sqp.ID),
-		true, utils.NonTransactional) // Temporary work around util proper cacheDataFromDB will be implemented
+	if err = dm.CacheDataFromDB(utils.StatQueueProfilePrefix, []string{sqp.TenantID()}, true); err != nil {
+		return
+	}
 	if withIndex {
 		indexer := NewReqFilterIndexer(dm, utils.StatQueueProfilePrefix, sqp.Tenant)
 		//remove old StatQueueProfile indexes
@@ -562,7 +583,13 @@ func (dm *DataManager) GetResource(tenant, id string, skipCache bool, transactio
 }
 
 func (dm *DataManager) SetResource(rs *Resource) (err error) {
-	return dm.DataDB().SetResourceDrv(rs)
+	if err = dm.DataDB().SetResourceDrv(rs); err != nil {
+		return
+	}
+	if err = dm.CacheDataFromDB(utils.ResourcesPrefix, []string{rs.TenantID()}, true); err != nil {
+		return
+	}
+	return
 }
 
 func (dm *DataManager) RemoveResource(tenant, id, transactionID string) (err error) {
@@ -599,7 +626,9 @@ func (dm *DataManager) SetResourceProfile(rp *ResourceProfile, withIndex bool) (
 	if err = dm.DataDB().SetResourceProfileDrv(rp); err != nil {
 		return err
 	}
-	cache.RemKey(utils.ResourceProfilesPrefix+utils.ConcatenatedKey(rp.Tenant, rp.ID), true, "") // ToDo: Remove here with autoreload
+	if err = dm.CacheDataFromDB(utils.ResourceProfilesPrefix, []string{rp.TenantID()}, true); err != nil {
+		return
+	}
 	//to be implemented in tests
 	if withIndex {
 		indexer := NewReqFilterIndexer(dm, utils.ResourceProfilesPrefix, rp.Tenant)
@@ -728,7 +757,6 @@ func (dm *DataManager) GetSharedGroup(key string, skipCache bool, transactionID
 	}
 	cache.Set(cachekey, sg, cacheCommit(transactionID), transactionID)
 	return
-
 }
 
 func (dm *DataManager) SetSharedGroup(sg *SharedGroup, transactionID string) (err error) {
@@ -752,7 +780,6 @@ func (dm *DataManager) RemoveSharedGroup(id, transactionID string) (err error) {
 	} else {
 		return dm.DataDB().RemoveSharedGroupDrv(id, transactionID)
 	}
-
 }
 
 func (dm *DataManager) SetLCR(lcr *LCR, transactionID string) (err error) {
@@ -1058,9 +1085,7 @@ func (dm *DataManager) SetSupplierProfile(supp *SupplierProfile, withIndex bool)
 	if err = dm.DataDB().SetSupplierProfileDrv(supp); err != nil {
 		return err
 	}
-	cache.RemKey(utils.SupplierProfilePrefix+utils.ConcatenatedKey(supp.Tenant, supp.ID), true, "")
-	ids := []string{supp.ID}
-	if err = dm.CacheDataFromDB(utils.SupplierProfilePrefix, ids, true); err != nil {
+	if err = dm.CacheDataFromDB(utils.SupplierProfilePrefix, []string{supp.TenantID()}, true); err != nil {
 		return
 	}
 	//to be implemented in tests
@@ -1140,7 +1165,7 @@ func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool)
 	if err = dm.DataDB().SetAttributeProfileDrv(ap); err != nil {
 		return err
 	}
-	if err = dm.CacheDataFromDB(utils.AttributeProfilePrefix, []string{ap.ID}, true); err != nil {
+	if err = dm.CacheDataFromDB(utils.AttributeProfilePrefix, []string{ap.TenantID()}, true); err != nil {
 		return
 	}
 	//to be implemented in tests

engine/loader_csv_test.go

@@ -298,9 +298,9 @@ cgrates.org,SPP_1,,,,,supplier1,FLTR_DST_DE,Account2,RPL_3,ResGroup3,Stat2,10,,
 cgrates.org,SPP_1,,,,,supplier1,,,,ResGroup4,Stat3,10,,
 `
 	attributeProfiles = `
-#,Tenant,ID,Context,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Append,Weight
+#Tenant,ID,Contexts,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Append,Weight
 cgrates.org,ALS1,con1,FLTR_1,2014-07-29T15:00:00Z,Field1,Initial1,Sub1,true,20
-cgrates.org,ALS1,,,,Field2,Initial2,Sub2,false,
+cgrates.org,ALS1,con2;con3,,,Field2,Initial2,Sub2,false,
 `
 )
 
@@ -1672,7 +1672,7 @@ func TestLoadAttributeProfiles(t *testing.T) {
 			TPid:      testTPID,
 			Tenant:    "cgrates.org",
 			ID:        "ALS1",
-			Contexts:  []string{"con1"},
+			Contexts:  []string{"con1", "con2", "con3"},
 			FilterIDs: []string{"FLTR_1"},
 			ActivationInterval: &utils.TPActivationInterval{
 				ActivationTime: "2014-07-29T15:00:00Z",

engine/onstor_it_test.go

@@ -1247,9 +1247,6 @@ func testOnStorITCacheAttributeProfile(t *testing.T) {
 		t.Errorf("Expected : %+v, but received %+v", expectedT, itm)
 	}
 
-	if _, hasIt := cache.Get(utils.AttributeProfilePrefix + attrProfile.TenantID()); hasIt {
-		t.Error("Already in cache")
-	}
 	if err := onStor.CacheDataFromDB(utils.AttributeProfilePrefix, []string{attrProfile.TenantID()}, false); err != nil {
 		t.Error(err)
 	}

engine/storage_mongo_datadb.go

@@ -25,7 +25,6 @@ import (
 	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
 	"io/ioutil"
-	//"log"
 	"strings"
 	"time"