Merge remote-tracking branch 'origin/master'

apier/v1/chargers_it_test.go

@@ -66,7 +66,7 @@ var sTestsCharger = []func(t *testing.T){
 	testChargerSResetStorDb,
 	testChargerSStartEngine,
 	testChargerSRPCConn,
-	testChargerSLoadFromFolder,
+	testChargerSLoadAddCharger,
 	testChargerSGetChargersForEvent,
 	testChargerSProcessEvent,
 	testChargerSSetChargerProfile,
@@ -138,13 +138,49 @@ func testChargerSRPCConn(t *testing.T) {
 	}
 }
 
-func testChargerSLoadFromFolder(t *testing.T) {
-	var reply string
-	attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tariffplans", "tutorial")}
-	if err := chargerRPC.Call("ApierV1.LoadTariffPlanFromFolder", attrs, &reply); err != nil {
-		t.Error(err)
+func testChargerSLoadAddCharger(t *testing.T) {
+	chargerProfile := &engine.ChargerProfile{
+		Tenant:    "cgrates.org",
+		ID:        "Charger1",
+		FilterIDs: []string{"*string:Account:1001"},
+		ActivationInterval: &utils.ActivationInterval{
+			ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC),
+		},
+		RunID:        "*default",
+		AttributeIDs: []string{"ATTR_1001_SIMPLEAUTH"},
+		Weight:       20,
+	}
+	var result string
+	if err := chargerRPC.Call("ApierV1.SetChargerProfile", chargerProfile, &result); err != nil {
+		t.Error(err)
+	} else if result != utils.OK {
+		t.Error("Unexpected reply returned", result)
+	}
+	alsPrf = &engine.AttributeProfile{
+		Tenant:   "cgrates.org",
+		ID:       "ATTR_1001_SIMPLEAUTH",
+		Contexts: []string{"simpleauth"},
+		Attributes: []*engine.Attribute{
+			&engine.Attribute{
+				FieldName: "Password",
+				Initial:   utils.ANY,
+				Substitute: config.RSRParsers{
+					&config.RSRParser{
+						Rules:           "CGRateS.org",
+						AllFiltersMatch: true,
+					},
+				},
+				Append: true,
+			},
+		},
+		Blocker: false,
+		Weight:  10,
+	}
+	if err := chargerRPC.Call("ApierV1.SetAttributeProfile", alsPrf, &result); err != nil {
+		t.Error(err)
+	} else if result != utils.OK {
+		t.Error("Unexpected reply returned", result)
 	}
-	time.Sleep(500 * time.Millisecond)
 }
 
 func testChargerSGetChargersForEvent(t *testing.T) {
@@ -156,7 +192,7 @@ func testChargerSGetChargersForEvent(t *testing.T) {
 			ActivationInterval: &utils.ActivationInterval{
 				ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC),
 			},
-			RunID:        "*rated",
+			RunID:        "*default",
 			AttributeIDs: []string{"ATTR_1001_SIMPLEAUTH"},
 			Weight:       20,
 		},
@@ -169,7 +205,7 @@ func testChargerSGetChargersForEvent(t *testing.T) {
 	if err := chargerRPC.Call(utils.ChargerSv1GetChargersForEvent, chargerEvent[0], &result); err != nil {
 		t.Error(err)
 	} else if !reflect.DeepEqual(result, chargerProfiles) {
-		t.Errorf("Expecting : %+v, received: %+v", chargerProfiles, result)
+		t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(chargerProfiles), utils.ToJSON(result))
 	}
 }
 
@@ -186,7 +222,7 @@ func testChargerSProcessEvent(t *testing.T) {
 				Event: map[string]interface{}{
 					utils.Account: "1001",
 					"Password":    "CGRateS.org",
-					"RunID":       "*rated",
+					"RunID":       "*default",
 				},
 			},
 		},

apier/v1/filter_indexes.go

@@ -323,7 +323,7 @@ func (self *ApierV1) computeThresholdIndexes(tenant string, thIDs *[]string,
 					},
 				}
 			} else if fltr, err = self.DataManager.GetFilter(th.Tenant, fltrID,
-				false, utils.NonTransactional); err != nil {
+				true, false, utils.NonTransactional); err != nil {
 				if err == utils.ErrNotFound {
 					err = fmt.Errorf("broken reference to filter: %+v for threshold: %+v",
 						fltrID, th)
@@ -391,7 +391,7 @@ func (self *ApierV1) computeAttributeIndexes(tenant, context string, attrIDs *[]
 					},
 				}
 			} else if fltr, err = self.DataManager.GetFilter(ap.Tenant, fltrID,
-				false, utils.NonTransactional); err != nil {
+				true, false, utils.NonTransactional); err != nil {
 				if err == utils.ErrNotFound {
 					err = fmt.Errorf("broken reference to filter: %+v for attribute: %+v",
 						fltrID, ap)
@@ -458,7 +458,7 @@ func (self *ApierV1) computeResourceIndexes(tenant string, rsIDs *[]string,
 					},
 				}
 			} else if fltr, err = self.DataManager.GetFilter(rp.Tenant, fltrID,
-				false, utils.NonTransactional); err != nil {
+				true, false, utils.NonTransactional); err != nil {
 				if err == utils.ErrNotFound {
 					err = fmt.Errorf("broken reference to filter: %+v for resource: %+v",
 						fltrID, rp)
@@ -525,7 +525,7 @@ func (self *ApierV1) computeStatIndexes(tenant string, stIDs *[]string,
 					},
 				}
 			} else if fltr, err = self.DataManager.GetFilter(sqp.Tenant, fltrID,
-				false, utils.NonTransactional); err != nil {
+				true, false, utils.NonTransactional); err != nil {
 				if err == utils.ErrNotFound {
 					err = fmt.Errorf("broken reference to filter: %+v for statqueue: %+v",
 						fltrID, sqp)
@@ -592,7 +592,7 @@ func (self *ApierV1) computeSupplierIndexes(tenant string, sppIDs *[]string,
 					},
 				}
 			} else if fltr, err = self.DataManager.GetFilter(spp.Tenant, fltrID,
-				false, utils.NonTransactional); err != nil {
+				true, false, utils.NonTransactional); err != nil {
 				if err == utils.ErrNotFound {
 					err = fmt.Errorf("broken reference to filter: %+v for suppliers: %+v",
 						fltrID, spp)
@@ -659,7 +659,7 @@ func (self *ApierV1) computeChargerIndexes(tenant string, cppIDs *[]string,
 					},
 				}
 			} else if fltr, err = self.DataManager.GetFilter(cpp.Tenant, fltrID,
-				false, utils.NonTransactional); err != nil {
+				true, false, utils.NonTransactional); err != nil {
 				if err == utils.ErrNotFound {
 					err = fmt.Errorf("broken reference to filter: %+v for charger: %+v",
 						fltrID, cpp)

apier/v1/filters.go

@@ -40,7 +40,7 @@ func (self *ApierV1) GetFilter(arg utils.TenantID, reply *engine.Filter) error {
 	if missing := utils.MissingStructFields(&arg, []string{"Tenant", "ID"}); len(missing) != 0 { //Params missing
 		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if fltr, err := self.DataManager.GetFilter(arg.Tenant, arg.ID, false, utils.NonTransactional); err != nil {
+	if fltr, err := self.DataManager.GetFilter(arg.Tenant, arg.ID, true, true, utils.NonTransactional); err != nil {
 		if err.Error() != utils.ErrNotFound.Error() {
 			err = utils.NewErrServerError(err)
 		}

apier/v1/filters_it_test.go

@@ -73,13 +73,6 @@ func TestFilterITMongo(t *testing.T) {
 	}
 }
 
-func TestFilterITPG(t *testing.T) {
-	filterConfigDIR = "tutpostgres"
-	for _, stest := range sTestsFilter {
-		t.Run(filterConfigDIR, stest)
-	}
-}
-
 func testFilterInitCfg(t *testing.T) {
 	var err error
 	filterCfgPath = path.Join(filterDataDir, "conf", "samples", filterConfigDIR)

apier/v1/sessions_thresholds_it_test.go

@@ -271,7 +271,7 @@ func TestSessionSv1ItInitiateSession(t *testing.T) {
 			utils.MONETARY: []*engine.Balance{
 				&engine.Balance{
 					//Uuid:  "c9a2c620-5256-483a-a92d-c51e94bb7667",
-					Value: 29.949000,
+					Value: 29.898000,
 					Directions: utils.StringMap{
 						"*out": true},
 					Weight: 10,
@@ -331,7 +331,7 @@ func TestSessionSv1ItTerminateSession(t *testing.T) {
 			utils.MONETARY: []*engine.Balance{
 				&engine.Balance{
 					//Uuid:  "c9a2c620-5256-483a-a92d-c51e94bb7667",
-					Value: 39.898000,
+					Value: 39.796000,
 					Directions: utils.StringMap{
 						"*out": true},
 					Weight: 10,

dispatcher/chargers_it_test.go

@@ -247,15 +247,12 @@ func testDspCppTestAuthKey2(t *testing.T) {
 	}
 	eChargers := &engine.ChargerProfiles{
 		&engine.ChargerProfile{
-			Tenant:    "cgrates.org",
-			ID:        "Charger1",
-			FilterIDs: []string{"*string:Account:1001"},
-			ActivationInterval: &utils.ActivationInterval{
-				ActivationTime: time.Date(2014, 7, 29, 15, 0, 0, 0, time.UTC),
-			},
-			RunID:        "*rated",
-			AttributeIDs: []string{"ATTR_1001_SIMPLEAUTH"},
-			Weight:       20,
+			Tenant:       "cgrates.org",
+			ID:           "DEFAULT",
+			FilterIDs:    []string{},
+			RunID:        "*default",
+			AttributeIDs: []string{"*none"},
+			Weight:       0,
 		},
 	}
 	var reply *engine.ChargerProfiles

dispatcher/sessions_it_test.go

@@ -391,6 +391,7 @@ func testDspSessionUpdate(t *testing.T) {
 				utils.SetupTime:   "2018-01-07T17:00:00Z",
 				utils.AnswerTime:  "2018-01-07T17:00:10Z",
 				utils.Usage:       300000000000.0,
+				"CGRID":           "5668666d6b8e44eb949042f25ce0796ec3592ff9",
 			},
 		},
 	}

engine/chargers_test.go

@@ -105,7 +105,7 @@ func TestChargerPopulateChargerService(t *testing.T) {
 		t.Errorf("Error: %+v", err)
 	}
 	chargerSrv, err = NewChargerService(dmCharger,
-		&FilterS{dm: dmAtr, cfg: defaultCfg}, nil, defaultCfg)
+		&FilterS{dm: dmCharger, cfg: defaultCfg}, nil, defaultCfg)
 	if err != nil {
 		t.Errorf("Error: %+v", err)
 	}
@@ -176,7 +176,7 @@ func TestChargerSetChargerProfiles(t *testing.T) {
 	//verify each charger from cache
 	for _, cp := range cPPs {
 		if tempCp, err := dmCharger.GetChargerProfile(cp.Tenant, cp.ID,
-			true, true, utils.NonTransactional); err != nil {
+			true, false, utils.NonTransactional); err != nil {
 			t.Errorf("Error: %+v", err)
 		} else if !reflect.DeepEqual(cp, tempCp) {
 			t.Errorf("Expecting: %+v, received: %+v", cp, tempCp)
@@ -190,19 +190,15 @@ func TestChargerMatchingChargerProfilesForEvent(t *testing.T) {
 		t.Errorf("Error: %+v", err)
 	}
 
-	rcv, err := chargerSrv.matchingChargerProfilesForEvent(chargerEvents[0])
-	if err != nil {
+	if rcv, err := chargerSrv.matchingChargerProfilesForEvent(chargerEvents[0]); err != nil {
 		t.Errorf("Error: %+v", err)
-	}
-	if !reflect.DeepEqual(cPPs[0], rcv[0]) {
+	} else if !reflect.DeepEqual(cPPs[0], rcv[0]) {
 		t.Errorf("Expecting: %+v, received: %+v ", cPPs[0], rcv[0])
 	}
 
-	rcv, err = chargerSrv.matchingChargerProfilesForEvent(chargerEvents[1])
-	if err != nil {
+	if rcv, err := chargerSrv.matchingChargerProfilesForEvent(chargerEvents[1]); err != nil {
 		t.Errorf("Error: %+v", err)
-	}
-	if !reflect.DeepEqual(cPPs[1], rcv[0]) {
+	} else if !reflect.DeepEqual(cPPs[1], rcv[0]) {
 		t.Errorf("Expecting: %+v, received: %+v", utils.ToJSON(cPPs[1]), utils.ToJSON(rcv))
 	}
 

engine/datamanager.go

@@ -235,7 +235,7 @@ func (dm *DataManager) CacheDataFromDB(prfx string, ids []string, mustBeCached b
 			_, err = dm.GetThreshold(tntID.Tenant, tntID.ID, false, true, utils.NonTransactional)
 		case utils.FilterPrefix:
 			tntID := utils.NewTenantID(dataID)
-			_, err = dm.GetFilter(tntID.Tenant, tntID.ID, true, utils.NonTransactional)
+			_, err = dm.GetFilter(tntID.Tenant, tntID.ID, false, true, utils.NonTransactional)
 		case utils.SupplierProfilePrefix:
 			tntID := utils.NewTenantID(dataID)
 			_, err = dm.GetSupplierProfile(tntID.Tenant, tntID.ID, false, true, utils.NonTransactional)
@@ -310,10 +310,10 @@ func (dm *DataManager) RemoveStatQueue(tenant, id string, transactionID string)
 }
 
 // GetFilter returns
-func (dm *DataManager) GetFilter(tenant, id string,
-	skipCache bool, transactionID string) (fltr *Filter, err error) {
+func (dm *DataManager) GetFilter(tenant, id string, cacheRead, cacheWrite bool,
+	transactionID string) (fltr *Filter, err error) {
 	tntID := utils.ConcatenatedKey(tenant, id)
-	if !skipCache {
+	if cacheRead {
 		if x, ok := Cache.Get(utils.CacheFilters, tntID); ok {
 			if x == nil {
 				return nil, utils.ErrNotFound
@@ -324,17 +324,21 @@ func (dm *DataManager) GetFilter(tenant, id string,
 	if strings.HasPrefix(id, utils.Meta) {
 		fltr, err = NewFilterFromInline(tenant, id)
 	} else {
-		fltr, err = dm.dataDB.GetFilterDrv(tenant, id)
+		fltr, err = dm.DataDB().GetFilterDrv(tenant, id)
 	}
 	if err != nil {
 		if err == utils.ErrNotFound {
-			Cache.Set(utils.CacheFilters, tntID, nil, nil,
-				cacheCommit(transactionID), transactionID)
+			if cacheWrite {
+				Cache.Set(utils.CacheFilters, tntID, nil, nil,
+					cacheCommit(transactionID), transactionID)
+			}
 		}
 		return nil, err
 	}
-	Cache.Set(utils.CacheFilters, tntID, fltr, nil,
-		cacheCommit(transactionID), transactionID)
+	if cacheWrite {
+		Cache.Set(utils.CacheFilters, tntID, fltr, nil,
+			cacheCommit(transactionID), transactionID)
+	}
 	return
 }
 

engine/filterindexer.go

@@ -224,7 +224,7 @@ func (rfi *FilterIndexer) RemoveItemFromIndex(tenant, itemID string, oldFilters
 				},
 			}
 		} else if fltr, err = rfi.dm.GetFilter(tenant, fltrID,
-			false, utils.NonTransactional); err != nil {
+			true, false, utils.NonTransactional); err != nil {
 			if err == utils.ErrNotFound {
 				err = fmt.Errorf("broken reference to filter: %+v for itemType: %+v and ID: %+v",
 					fltrID, rfi.itemType, itemID)
@@ -285,7 +285,7 @@ func createAndIndex(itemPrefix, tenant, context, itemID string, filterIDs []stri
 				},
 			}
 		} else if fltr, err = dm.GetFilter(tenant, fltrID,
-			false, utils.NonTransactional); err != nil {
+			true, false, utils.NonTransactional); err != nil {
 			if err == utils.ErrNotFound {
 				err = fmt.Errorf("broken reference to filter: %+v for itemType: %+v and ID: %+v",
 					fltrID, itemPrefix, itemID)

engine/filters.go

@@ -85,7 +85,7 @@ func (fS *FilterS) Pass(tenant string, filterIDs []string,
 	}
 	for _, fltrID := range filterIDs {
 		f, err := fS.dm.GetFilter(tenant, fltrID,
-			false, utils.NonTransactional)
+			true, true, utils.NonTransactional)
 		if err != nil {
 			return false, err
 		}

engine/onstor_it_test.go

@@ -2283,7 +2283,7 @@ func testOnStorITFilter(t *testing.T) {
 		},
 	}
 	if _, rcvErr := onStor.GetFilter("cgrates.org", "Filter1",
-		false, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound {
+		true, false, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound {
 		t.Error(rcvErr)
 	}
 	if err := onStor.SetFilter(fp); err != nil {
@@ -2291,14 +2291,14 @@ func testOnStorITFilter(t *testing.T) {
 	}
 	//get from cache
 	if rcv, err := onStor.GetFilter("cgrates.org", "Filter1",
-		false, utils.NonTransactional); err != nil {
+		true, false, utils.NonTransactional); err != nil {
 		t.Error(err)
 	} else if !(reflect.DeepEqual(fp, rcv)) {
 		t.Errorf("Expecting: %v, received: %v", fp, rcv)
 	}
 	//get from database
 	if rcv, err := onStor.GetFilter("cgrates.org", "Filter1",
-		true, utils.NonTransactional); err != nil {
+		false, false, utils.NonTransactional); err != nil {
 		t.Error(err)
 	} else if !(reflect.DeepEqual(fp, rcv)) {
 		t.Errorf("Expecting: %v, received: %v", fp, rcv)
@@ -2328,14 +2328,14 @@ func testOnStorITFilter(t *testing.T) {
 	time.Sleep(sleepDelay)
 	//get from cache
 	if rcv, err := onStor.GetFilter("cgrates.org", "Filter1",
-		false, utils.NonTransactional); err != nil {
+		true, false, utils.NonTransactional); err != nil {
 		t.Error(err)
 	} else if !(reflect.DeepEqual(fp, rcv)) {
 		t.Errorf("Expecting: %v, received: %v", fp, rcv)
 	}
 	//get from database
 	if rcv, err := onStor.GetFilter("cgrates.org", "Filter1",
-		true, utils.NonTransactional); err != nil {
+		false, false, utils.NonTransactional); err != nil {
 		t.Error(err)
 	} else if !(reflect.DeepEqual(fp, rcv)) {
 		t.Errorf("Expecting: %v, received: %v", fp, rcv)
@@ -2345,12 +2345,12 @@ func testOnStorITFilter(t *testing.T) {
 	}
 	//check cache if removed
 	if _, rcvErr := onStor.GetFilter("cgrates.org", "Filter1",
-		false, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound {
+		true, false, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound {
 		t.Error(rcvErr)
 	}
 	//check database if removed
 	if _, rcvErr := onStor.GetFilter("cgrates.org", "Filter1",
-		true, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound {
+		false, false, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound {
 		t.Error(rcvErr)
 	}
 }

loaders/loader_test.go

@@ -454,14 +454,14 @@ cgrates.org,FLTR_ACNT_1002,*string,Account,1002,2014-07-29T15:00:00Z
 		t.Errorf("wrong buffer content: %+v", ldr.bufLoaderData)
 	}
 	if fltr, err := ldr.dm.GetFilter("cgrates.org", "FLTR_1",
-		false, utils.NonTransactional); err != nil {
+		true, true, utils.NonTransactional); err != nil {
 		t.Error(err)
 	} else if !reflect.DeepEqual(eFltr1, fltr) {
 		t.Errorf("expecting: %s, received: %s",
 			utils.ToJSON(eFltr1), utils.ToJSON(fltr))
 	}
 	if fltr, err := ldr.dm.GetFilter("cgrates.org", "FLTR_DST_DE",
-		false, utils.NonTransactional); err != nil {
+		true, true, utils.NonTransactional); err != nil {
 		t.Error(err)
 	} else if !reflect.DeepEqual(eFltr2, fltr) {
 		t.Errorf("expecting: %s, received: %s",

migrator/filters.go

@@ -36,7 +36,7 @@ func (m *Migrator) migrateCurrentRequestFilter() (err error) {
 	}
 	for _, id := range ids {
 		idg := strings.TrimPrefix(id, utils.FilterPrefix+tenant+":")
-		fl, err := m.dmIN.DataManager().GetFilter(tenant, idg, true, utils.NonTransactional)
+		fl, err := m.dmIN.DataManager().GetFilter(tenant, idg, false, false, utils.NonTransactional)
 		if err != nil {
 			return err
 		}

migrator/stats.go

@@ -71,6 +71,7 @@ func (m *Migrator) migrateCurrentStats() (err error) {
 		idg := strings.TrimPrefix(id, utils.StatQueuePrefix+tenant+":")
 		sgs, err := m.dmIN.DataManager().GetStatQueue(tenant, idg, false, false, utils.NonTransactional)
 		if err != nil {
+
 			return err
 		}
 		if sgs != nil {

migrator/stats_it_test.go

@@ -263,7 +263,7 @@ func testStsITMigrateAndMove(t *testing.T) {
 			t.Errorf("Expecting: %+v, received: %+v", sqp, result)
 		}
 
-		result1, err := stsMigrator.dmOut.DataManager().DataDB().GetFilterDrv("cgrates.org", v1Sts.Id)
+		result1, err := stsMigrator.dmOut.DataManager().GetFilter("cgrates.org", v1Sts.Id, false, false, utils.NonTransactional)
 		if err != nil {
 			t.Error("Error when getting Stats ", err.Error())
 		}
@@ -282,12 +282,15 @@ func testStsITMigrateAndMove(t *testing.T) {
 		}
 
 	case utils.Move:
-		if err := stsMigrator.dmIN.DataManager().DataDB().SetStatQueueProfileDrv(sqp); err != nil {
+		if err := stsMigrator.dmIN.DataManager().SetStatQueueProfile(sqp, false); err != nil {
 			t.Error("Error when setting Stats ", err.Error())
 		}
 		if err := stsMigrator.dmIN.DataManager().SetStatQueue(sq); err != nil {
 			t.Error("Error when setting Stats ", err.Error())
 		}
+		if err := stsMigrator.dmOut.DataManager().SetFilter(filter); err != nil {
+			t.Error("Error when setting Filter ", err.Error())
+		}
 		currentVersion := engine.CurrentDataDBVersions()
 		err := stsMigrator.dmOut.DataManager().DataDB().SetVersions(currentVersion, false)
 		if err != nil {