started rating profiles data validation on load

apier/v1/apier.go

@@ -327,9 +327,10 @@ func (self *ApierV1) LoadCdrStats(attrs AttrLoadCdrStats, reply *string) error {
 }
 
 type AttrLoadTpFromStorDb struct {
-	TPid    string
-	FlushDb bool // Flush ratingDb before loading
-	DryRun  bool // Only simulate, no write
+	TPid     string
+	FlushDb  bool // Flush ratingDb before loading
+	DryRun   bool // Only simulate, no write
+	Validate bool // Run structural checks
 }
 
 // Loads complete data in a TP from storDb
@@ -341,8 +342,14 @@ func (self *ApierV1) LoadTariffPlanFromStorDb(attrs AttrLoadTpFromStorDb, reply
 	if err := dbReader.LoadAll(); err != nil {
 		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
 	}
+	if attrs.Validate {
+		if !dbReader.IsDataValid() {
+			*reply = OK
+			return errors.New("invalid data")
+		}
+	}
 	if attrs.DryRun {
-		*reply = "OK"
+		*reply = OK
 		return nil // Mission complete, no errors
 	}
 	if err := dbReader.WriteToDatabase(attrs.FlushDb, false); err != nil {
@@ -978,9 +985,17 @@ func (self *ApierV1) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder,
 		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
 	}
 	if attrs.DryRun {
-		*reply = "OK"
+		*reply = OK
 		return nil // Mission complete, no errors
 	}
+
+	if attrs.Validate {
+		if !loader.IsDataValid() {
+			*reply = OK
+			return errors.New("invalid data")
+		}
+	}
+
 	if err := loader.WriteToDatabase(attrs.FlushDb, false); err != nil {
 		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
 	}

cmd/cgr-loader/cgr-loader.go

@@ -63,6 +63,7 @@ var (
 	version         = flag.Bool("version", false, "Prints the application version.")
 	verbose         = flag.Bool("verbose", false, "Enable detailed verbose logging output")
 	dryRun          = flag.Bool("dry_run", false, "When true will not save loaded data to dataDb but just parse it for consistency and errors.")
+	validate        = flag.Bool("validate", false, "When true will run various check on the loaded data to check for structural errors")
 	stats           = flag.Bool("stats", false, "Generates statsistics about given data.")
 	fromStorDb      = flag.Bool("from_stordb", false, "Load the tariff plan from storDb to dataDb")
 	toStorDb        = flag.Bool("to_stordb", false, "Import the tariff plan from files to storDb")
@@ -162,6 +163,11 @@ func main() {
 	if *stats {
 		loader.ShowStatistics()
 	}
+	if *validate {
+		if !loader.IsDataValid() {
+			return
+		}
+	}
 	if *dryRun { // We were just asked to parse the data, not saving it
 		return
 	}

engine/loader_csv.go

@@ -179,6 +179,17 @@ func (csvr *CSVReader) ShowStatistics() {
 	log.Print("CDR stats: ", len(csvr.cdrStats))
 }
 
+func (csvr *CSVReader) IsDataValid() bool {
+	valid := true
+	for rplTag, rpl := range csvr.ratingPlans {
+		if !rpl.IsValid() {
+			log.Printf("The rating plan %s is not covering all weekdays", rplTag)
+			valid = false
+		}
+	}
+	return valid
+}
+
 func (csvr *CSVReader) WriteToDatabase(flush, verbose bool) (err error) {
 	dataStorage := csvr.dataStorage
 	accountingStorage := csvr.accountingStorage

engine/loader_db.go

@@ -132,6 +132,17 @@ func (dbr *DbReader) ShowStatistics() {
 	log.Print("LCR rules: ", len(dbr.lcrs))
 }
 
+func (dbr *DbReader) IsDataValid() bool {
+	valid := true
+	for rplTag, rpl := range dbr.ratingPlans {
+		if !rpl.IsValid() {
+			log.Printf("The rating plan %s is not covering all weekdays", rplTag)
+			valid = false
+		}
+	}
+	return valid
+}
+
 func (dbr *DbReader) WriteToDatabase(flush, verbose bool) (err error) {
 	storage := dbr.dataDb
 	if flush {

engine/loader_helpers.go

@@ -119,6 +119,7 @@ type TPLoader interface {
 	LoadAll() error
 	GetLoadedIds(string) ([]string, error)
 	ShowStatistics()
+	IsDataValid() bool
 	WriteToDatabase(bool, bool) error
 }
 

engine/rateinterval.go

@@ -167,11 +167,19 @@ func (rit *RITiming) IsActiveAt(t time.Time, endTime bool) bool {
 	return true
 }
 
-// Returns wheter the Timing is active now
+// IsActive returns wheter the Timing is active now
 func (rit *RITiming) IsActive() bool {
 	return rit.IsActiveAt(time.Now(), false)
 }
 
+func (rit *RITiming) IsBlank() bool {
+	return len(rit.Years) == 0 &&
+		len(rit.Months) == 0 &&
+		len(rit.MonthDays) == 0 &&
+		len(rit.WeekDays) == 0 &&
+		rit.StartTime == "00:00:00"
+}
+
 func (rit *RITiming) Stringify() string {
 	return utils.Sha1(fmt.Sprintf("%v", rit))[:8]
 }

engine/ratingplan.go

@@ -112,3 +112,37 @@ func (rp *RatingPlan) GetHistoryRecord() history.Record {
 		Payload:  js,
 	}
 }
+
+// IsValid determines if the rating plan covers a continous period of time
+func (rp *RatingPlan) IsValid() bool {
+	weekdays := make([]int, 7)
+	for _, tm := range rp.Timings {
+		// if it is a blank timing than it will match all
+		if tm.IsBlank() {
+			return true
+		}
+		// skip the special timings (for specific dates)
+		if len(tm.Years) != 0 || len(tm.Months) != 0 || len(tm.MonthDays) != 0 {
+			continue
+		}
+		// if the startime is not midnight than is an extra time
+		if tm.StartTime != "00:00:00" {
+			continue
+		}
+		//check if all weekdays are covered
+		for _, wd := range tm.WeekDays {
+			weekdays[wd-1] = 1
+		}
+		allWeekdaysCovered := true
+		for _, wd := range weekdays {
+			if wd != 1 {
+				allWeekdaysCovered = false
+				break
+			}
+		}
+		if allWeekdaysCovered {
+			return true
+		}
+	}
+	return false
+}

engine/ratingplan_test.go

@@ -218,6 +218,93 @@ func TestGetActiveForCall(t *testing.T) {
 	}
 }
 
+func TestRatingPlanIsValidEmpty(t *testing.T) {
+	rpl := &RatingPlan{}
+	if rpl.IsValid() {
+		t.Errorf("Error determining rating plan's valididty: %+v", rpl)
+	}
+}
+
+func TestRatingPlanIsValidBlank(t *testing.T) {
+	rpl := &RatingPlan{
+		Timings: map[string]*RITiming{
+			"blank": &RITiming{StartTime: "00:00:00"},
+			"other": &RITiming{WeekDays: utils.WeekDays{1, 2, 3}, StartTime: "00:00:00"},
+		},
+	}
+	if !rpl.IsValid() {
+		t.Errorf("Error determining rating plan's valididty: %+v", rpl)
+	}
+}
+
+func TestRatingPlanIsValidGood(t *testing.T) {
+	rpl := &RatingPlan{
+		Timings: map[string]*RITiming{
+			"first":  &RITiming{WeekDays: utils.WeekDays{1, 2, 3}, StartTime: "00:00:00"},
+			"second": &RITiming{WeekDays: utils.WeekDays{4, 5, 6}, StartTime: "00:00:00"},
+			"third":  &RITiming{WeekDays: utils.WeekDays{7}, StartTime: "00:00:00"},
+		},
+	}
+	if !rpl.IsValid() {
+		t.Errorf("Error determining rating plan's valididty: %+v", rpl)
+	}
+}
+
+func TestRatingPlanIsValidBad(t *testing.T) {
+	rpl := &RatingPlan{
+		Timings: map[string]*RITiming{
+			"first":  &RITiming{WeekDays: utils.WeekDays{1, 2, 3}, StartTime: "00:00:00"},
+			"second": &RITiming{WeekDays: utils.WeekDays{4, 5, 7}, StartTime: "00:00:00"},
+		},
+	}
+	if rpl.IsValid() {
+		t.Errorf("Error determining rating plan's valididty: %+v", rpl)
+	}
+}
+
+func TestRatingPlanIsValidSpecial(t *testing.T) {
+	rpl := &RatingPlan{
+		Timings: map[string]*RITiming{
+			"special": &RITiming{Years: utils.Years{2015}, Months: utils.Months{5}, MonthDays: utils.MonthDays{1}, StartTime: "00:00:00"},
+			"first":   &RITiming{WeekDays: utils.WeekDays{1, 2, 3}, StartTime: "00:00:00"},
+			"second":  &RITiming{WeekDays: utils.WeekDays{4, 5, 6}, StartTime: "00:00:00"},
+			"third":   &RITiming{WeekDays: utils.WeekDays{7}, StartTime: "00:00:00"},
+		},
+	}
+	if !rpl.IsValid() {
+		t.Errorf("Error determining rating plan's valididty: %+v", rpl)
+	}
+}
+
+func TestRatingPlanIsValidMultiple(t *testing.T) {
+	rpl := &RatingPlan{
+		Timings: map[string]*RITiming{
+			"special":  &RITiming{Years: utils.Years{2015}, Months: utils.Months{5}, MonthDays: utils.MonthDays{1}, StartTime: "00:00:00"},
+			"first":    &RITiming{WeekDays: utils.WeekDays{1, 2, 3}, StartTime: "00:00:00"},
+			"first_08": &RITiming{WeekDays: utils.WeekDays{1, 2, 3}, StartTime: "08:00:00"},
+			"second":   &RITiming{WeekDays: utils.WeekDays{4, 5, 6}, StartTime: "00:00:00"},
+			"third":    &RITiming{WeekDays: utils.WeekDays{7}, StartTime: "00:00:00"},
+		},
+	}
+	if !rpl.IsValid() {
+		t.Errorf("Error determining rating plan's valididty: %+v", rpl)
+	}
+}
+
+func TestRatingPlanIsValidMissing(t *testing.T) {
+	rpl := &RatingPlan{
+		Timings: map[string]*RITiming{
+			"special":  &RITiming{Years: utils.Years{2015}, Months: utils.Months{5}, MonthDays: utils.MonthDays{1}, StartTime: "00:00:00"},
+			"first_08": &RITiming{WeekDays: utils.WeekDays{1, 2, 3}, StartTime: "08:00:00"},
+			"second":   &RITiming{WeekDays: utils.WeekDays{4, 5, 6}, StartTime: "00:00:00"},
+			"third":    &RITiming{WeekDays: utils.WeekDays{7}, StartTime: "00:00:00"},
+		},
+	}
+	if rpl.IsValid() {
+		t.Errorf("Error determining rating plan's valididty: %+v", rpl)
+	}
+}
+
 /**************************** Benchmarks *************************************/
 
 func BenchmarkRatingPlanMarshalJson(b *testing.B) {

utils/apitpdata.go

@@ -855,6 +855,7 @@ type AttrLoadTpFromFolder struct {
 	FolderPath string // Take files from folder absolute path
 	DryRun     bool   // Do not write to database but parse only
 	FlushDb    bool   // Flush previous data before loading new one
+	Validate   bool   // Run structural checks on data
 }
 
 type AttrGetDestination struct {

utils/dateseries.go

@@ -20,6 +20,7 @@ package utils
 
 import (
 	"fmt"
+	"reflect"
 	"sort"
 	"strconv"
 	"strings"
@@ -148,6 +149,12 @@ func (m Months) Serialize(sep string) string {
 	return mStr
 }
 
+func (m Months) IsComplete() bool {
+	allMonths := Months{time.January, time.February, time.March, time.April, time.May, time.June, time.July, time.August, time.September, time.October, time.November, time.December}
+	m.Sort()
+	return reflect.DeepEqual(m, allMonths)
+}
+
 // Defines month days series
 type MonthDays []int
 

utils/dateseries_test.go

@@ -25,7 +25,7 @@ import (
 	"time"
 )
 
-func TestMonthStoreRestoreJson(t *testing.T) {
+func TestDateseriesMonthStoreRestoreJson(t *testing.T) {
 	m := Months{5, 6, 7, 8}
 	r, _ := json.Marshal(m)
 	if string(r) != "[5,6,7,8]" {
@@ -38,7 +38,7 @@ func TestMonthStoreRestoreJson(t *testing.T) {
 	}
 }
 
-func TestMonthDayStoreRestoreJson(t *testing.T) {
+func TestDateseriesMonthDayStoreRestoreJson(t *testing.T) {
 	md := MonthDays{24, 25, 26}
 	r, _ := json.Marshal(md)
 	if string(r) != "[24,25,26]" {
@@ -51,7 +51,7 @@ func TestMonthDayStoreRestoreJson(t *testing.T) {
 	}
 }
 
-func TestWeekDayStoreRestoreJson(t *testing.T) {
+func TestDateseriesWeekDayStoreRestoreJson(t *testing.T) {
 	wd := WeekDays{time.Saturday, time.Sunday}
 	r, _ := json.Marshal(wd)
 	if string(r) != "[6,0]" {
@@ -64,7 +64,7 @@ func TestWeekDayStoreRestoreJson(t *testing.T) {
 	}
 }
 
-func TestYearsSerialize(t *testing.T) {
+func TestDateseriesYearsSerialize(t *testing.T) {
 	ys := &Years{}
 	yString := ys.Serialize(";")
 	expectString := "*any"
@@ -85,7 +85,7 @@ func TestYearsSerialize(t *testing.T) {
 	}
 }
 
-func TestMonthsSerialize(t *testing.T) {
+func TestDateseriesMonthsSerialize(t *testing.T) {
 	mths := &Months{}
 	mString := mths.Serialize(";")
 	expectString := "*any"
@@ -106,7 +106,7 @@ func TestMonthsSerialize(t *testing.T) {
 	}
 }
 
-func TestMonthDaysSerialize(t *testing.T) {
+func TestDateseriesMonthDaysSerialize(t *testing.T) {
 	mds := &MonthDays{}
 	mdsString := mds.Serialize(";")
 	expectString := "*any"
@@ -127,7 +127,7 @@ func TestMonthDaysSerialize(t *testing.T) {
 	}
 }
 
-func TestWeekDaysSerialize(t *testing.T) {
+func TestDateseriesWeekDaysSerialize(t *testing.T) {
 	wds := &WeekDays{}
 	wdsString := wds.Serialize(";")
 	expectString := "*any"
@@ -147,3 +147,17 @@ func TestWeekDaysSerialize(t *testing.T) {
 		t.Errorf("Expected: %s, got: %s", expectString3, wdsString3)
 	}
 }
+
+func TestDateseriesMonthsIsCompleteNot(t *testing.T) {
+	months := Months{time.January, time.February, time.March, time.April, time.May, time.June, time.July, time.August, time.September, time.October, time.November}
+	if months.IsComplete() {
+		t.Error("Error months IsComplete: ", months)
+	}
+}
+
+func TestDateseriesMonthsIsCompleteYes(t *testing.T) {
+	months := Months{time.January, time.February, time.March, time.April, time.May, time.June, time.July, time.August, time.September, time.October, time.November, time.December}
+	if !months.IsComplete() {
+		t.Error("Error months IsComplete: ", months)
+	}
+}