Remove RateS

This commit is contained in:
ionutboangiu
2021-03-31 11:46:15 +03:00
committed by Dan Christian Bogos
parent cc39a99a2c
commit 3197b3a458
78 changed files with 1583 additions and 18512 deletions

View File

@@ -143,8 +143,6 @@ func (m *Migrator) Migrate(taskIDs []string) (err error, stats map[string]int) {
err = m.migrateTimings()
case utils.MetaResources:
err = m.migrateResources()
case utils.MetaRateProfiles:
err = m.migrateRateProfiles()
case MetaAliases:
err = m.migrateAlias()
case utils.MetaUsers:
@@ -182,8 +180,6 @@ func (m *Migrator) Migrate(taskIDs []string) (err error, stats map[string]int) {
err = m.migrateTPsharedgroups()
case utils.MetaTpRatingProfiles:
err = m.migrateTPratingprofiles()
case utils.MetaTpRateProfiles:
err = m.migrateTPRateProfiles()
case utils.MetaTpActionProfiles:
err = m.migrateTPActionProfiles()
case utils.MetaTpResources:

View File

@@ -1,95 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package migrator
import (
"fmt"
"strings"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
func (m *Migrator) migrateCurrentRateProfiles() (err error) {
var ids []string
ids, err = m.dmIN.DataManager().DataDB().GetKeysForPrefix(utils.RateProfilePrefix)
if err != nil {
return err
}
for _, id := range ids {
tntID := strings.SplitN(strings.TrimPrefix(id, utils.RateProfilePrefix), utils.InInFieldSep, 2)
if len(tntID) < 2 {
return fmt.Errorf("Invalid key <%s> when migrating rate profiles", id)
}
rp, err := m.dmIN.DataManager().GetRateProfile(tntID[0], tntID[1], false, false, utils.NonTransactional)
if err != nil {
return err
}
if rp == nil || m.dryRun {
continue
}
if err := m.dmOut.DataManager().SetRateProfile(rp, true); err != nil {
return err
}
if err := m.dmIN.DataManager().RemoveRateProfile(tntID[0], tntID[1], utils.NonTransactional, false); err != nil {
return err
}
m.stats[utils.RateProfiles]++
}
return
}
func (m *Migrator) migrateRateProfiles() (err error) {
var vrs engine.Versions
current := engine.CurrentDataDBVersions()
if vrs, err = m.getVersions(utils.RateProfiles); err != nil {
return
}
migrated := true
for {
version := vrs[utils.RateProfiles]
for {
switch version {
default:
return fmt.Errorf("Unsupported version %v", version)
case current[utils.RateProfiles]:
migrated = false
if m.sameDataDB {
break
}
if err = m.migrateCurrentRateProfiles(); err != nil {
return
}
}
if version == current[utils.RateProfiles] || err == utils.ErrNoMoreData {
break
}
}
if err == utils.ErrNoMoreData || !migrated {
break
}
m.stats[utils.RateProfiles]++
}
// All done, update version wtih current one
if err = m.setVersions(utils.RateProfiles); err != nil {
return
}
return m.ensureIndexesDataDB(engine.ColRpp)
}

View File

@@ -1,275 +0,0 @@
// +build integration
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package migrator
import (
"log"
"path"
"reflect"
"testing"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
var (
ratePrfPathIn string
ratePrfPathOut string
ratePrfCfgIn *config.CGRConfig
ratePrfCfgOut *config.CGRConfig
ratePrfMigrator *Migrator
ratePrfAction string
)
var sTestsRatePrfIT = []func(t *testing.T){
testRatePrfITConnect,
testRatePrfITFlush,
testRatePrfITMigrateAndMove,
}
func TestRatePrfITMove1(t *testing.T) {
var err error
ratePrfPathIn = path.Join(*dataDir, "conf", "samples", "tutmongo")
ratePrfCfgIn, err = config.NewCGRConfigFromPath(ratePrfPathIn)
if err != nil {
t.Fatal(err)
}
ratePrfPathOut = path.Join(*dataDir, "conf", "samples", "tutmysql")
ratePrfCfgOut, err = config.NewCGRConfigFromPath(ratePrfPathOut)
if err != nil {
t.Fatal(err)
}
ratePrfAction = utils.Move
for _, stest := range sTestsRatePrfIT {
t.Run("TestRatePrfITMove", stest)
}
ratePrfMigrator.Close()
}
func TestRatePrfITMove2(t *testing.T) {
var err error
ratePrfPathIn = path.Join(*dataDir, "conf", "samples", "tutmysql")
ratePrfCfgIn, err = config.NewCGRConfigFromPath(ratePrfPathIn)
if err != nil {
t.Fatal(err)
}
ratePrfPathOut = path.Join(*dataDir, "conf", "samples", "tutmongo")
ratePrfCfgOut, err = config.NewCGRConfigFromPath(ratePrfPathOut)
if err != nil {
t.Fatal(err)
}
ratePrfAction = utils.Move
for _, stest := range sTestsRatePrfIT {
t.Run("TestRatePrfITMove", stest)
}
ratePrfMigrator.Close()
}
func TestRatePrfITMoveEncoding(t *testing.T) {
var err error
ratePrfPathIn = path.Join(*dataDir, "conf", "samples", "tutmongo")
ratePrfCfgIn, err = config.NewCGRConfigFromPath(ratePrfPathIn)
if err != nil {
t.Fatal(err)
}
ratePrfPathOut = path.Join(*dataDir, "conf", "samples", "tutmongojson")
ratePrfCfgOut, err = config.NewCGRConfigFromPath(ratePrfPathOut)
if err != nil {
t.Fatal(err)
}
ratePrfAction = utils.Move
for _, stest := range sTestsRatePrfIT {
t.Run("TestRatePrfITMoveEncoding", stest)
}
ratePrfMigrator.Close()
}
func TestRatePrfITMoveEncoding2(t *testing.T) {
var err error
ratePrfPathIn = path.Join(*dataDir, "conf", "samples", "tutmysql")
ratePrfCfgIn, err = config.NewCGRConfigFromPath(ratePrfPathIn)
if err != nil {
t.Fatal(err)
}
ratePrfPathOut = path.Join(*dataDir, "conf", "samples", "tutmysqljson")
ratePrfCfgOut, err = config.NewCGRConfigFromPath(ratePrfPathOut)
if err != nil {
t.Fatal(err)
}
ratePrfAction = utils.Move
for _, stest := range sTestsRatePrfIT {
t.Run("TestRatePrfITMoveEncoding2", stest)
}
ratePrfMigrator.Close()
}
func testRatePrfITConnect(t *testing.T) {
dataDBIn, err := NewMigratorDataDB(ratePrfCfgIn.DataDbCfg().Type,
ratePrfCfgIn.DataDbCfg().Host, ratePrfCfgIn.DataDbCfg().Port,
ratePrfCfgIn.DataDbCfg().Name, ratePrfCfgIn.DataDbCfg().User,
ratePrfCfgIn.DataDbCfg().Password, ratePrfCfgIn.GeneralCfg().DBDataEncoding,
config.CgrConfig().CacheCfg(), ratePrfCfgIn.DataDbCfg().Opts)
if err != nil {
log.Fatal(err)
}
dataDBOut, err := NewMigratorDataDB(ratePrfCfgOut.DataDbCfg().Type,
ratePrfCfgOut.DataDbCfg().Host, ratePrfCfgOut.DataDbCfg().Port,
ratePrfCfgOut.DataDbCfg().Name, ratePrfCfgOut.DataDbCfg().User,
ratePrfCfgOut.DataDbCfg().Password, ratePrfCfgOut.GeneralCfg().DBDataEncoding,
config.CgrConfig().CacheCfg(), ratePrfCfgOut.DataDbCfg().Opts)
if err != nil {
log.Fatal(err)
}
if reflect.DeepEqual(ratePrfPathIn, ratePrfPathOut) {
ratePrfMigrator, err = NewMigrator(dataDBIn, dataDBOut, nil, nil,
false, true, false, false)
} else {
ratePrfMigrator, err = NewMigrator(dataDBIn, dataDBOut, nil, nil,
false, false, false, false)
}
if err != nil {
log.Fatal(err)
}
}
func testRatePrfITFlush(t *testing.T) {
if err := ratePrfMigrator.dmOut.DataManager().DataDB().Flush(""); err != nil {
t.Error(err)
}
if isEmpty, err := ratePrfMigrator.dmOut.DataManager().DataDB().IsDBEmpty(); err != nil {
t.Error(err)
} else if isEmpty != true {
t.Errorf("Expecting: true got :%+v", isEmpty)
}
if err := engine.SetDBVersions(ratePrfMigrator.dmOut.DataManager().DataDB()); err != nil {
t.Error("Error ", err.Error())
}
if err := ratePrfMigrator.dmIN.DataManager().DataDB().Flush(""); err != nil {
t.Error(err)
}
if isEmpty, err := ratePrfMigrator.dmIN.DataManager().DataDB().IsDBEmpty(); err != nil {
t.Error(err)
} else if isEmpty != true {
t.Errorf("Expecting: true got :%+v", isEmpty)
}
if err := engine.SetDBVersions(ratePrfMigrator.dmIN.DataManager().DataDB()); err != nil {
t.Error("Error ", err.Error())
}
}
func testRatePrfITMigrateAndMove(t *testing.T) {
minDec, err := utils.NewDecimalFromUsage("1m")
if err != nil {
t.Error(err)
}
secDec, err := utils.NewDecimalFromUsage("1s")
if err != nil {
t.Error(err)
}
rPrf := &utils.RateProfile{
Tenant: "cgrates.org",
ID: "RP1",
FilterIDs: []string{"*string:~*req.Subject:1001"},
Weights: utils.DynamicWeights{
{
Weight: 0,
},
},
MinCost: utils.NewDecimal(1, 1),
MaxCost: utils.NewDecimal(6, 1),
MaxCostStrategy: "*free",
Rates: map[string]*utils.Rate{
"FIRST_GI": {
ID: "FIRST_GI",
FilterIDs: []string{"*gi:~*req.Usage:0"},
Weights: utils.DynamicWeights{
{
Weight: 0,
},
},
IntervalRates: []*utils.IntervalRate{
{
RecurrentFee: utils.NewDecimal(12, 2),
Unit: minDec,
Increment: minDec,
},
},
Blocker: false,
},
"SECOND_GI": {
ID: "SECOND_GI",
FilterIDs: []string{"*gi:~*req.Usage:1m"},
Weights: utils.DynamicWeights{
{
Weight: 10,
},
},
IntervalRates: []*utils.IntervalRate{
{
RecurrentFee: utils.NewDecimal(6, 2),
Unit: minDec,
Increment: secDec,
},
},
Blocker: false,
},
},
}
if err := rPrf.Compile(); err != nil {
t.Fatal(err)
}
switch ratePrfAction {
case utils.Migrate: //QQ for the moment only one version of rate profiles exists
case utils.Move:
if err := ratePrfMigrator.dmIN.DataManager().SetRateProfile(rPrf, true); err != nil {
t.Error(err)
}
currentVersion := engine.CurrentDataDBVersions()
err := ratePrfMigrator.dmIN.DataManager().DataDB().SetVersions(currentVersion, false)
if err != nil {
t.Error("Error when setting version for RatePrf ", err.Error())
}
_, err = ratePrfMigrator.dmOut.DataManager().GetRateProfile("cgrates.org", "RP1", false, false, utils.NonTransactional)
if err != utils.ErrNotFound {
t.Error(err)
}
err, _ = ratePrfMigrator.Migrate([]string{utils.MetaRateProfiles})
if err != nil {
t.Error("Error when migrating RatePrf ", err.Error())
}
ratePrfult, err := ratePrfMigrator.dmOut.DataManager().GetRateProfile("cgrates.org", "RP1", false, false, utils.NonTransactional)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(ratePrfult, rPrf) {
t.Errorf("Expecting: %+v, received: %+v", rPrf, ratePrfult)
}
ratePrfult, err = ratePrfMigrator.dmIN.DataManager().GetRateProfile("cgrates.org", "RP1", false, false, utils.NonTransactional)
if err != utils.ErrNotFound {
t.Error(err)
} else if ratePrfMigrator.stats[utils.RateProfiles] != 1 {
t.Errorf("Expected 1, received: %v", ratePrfMigrator.stats[utils.RateProfiles])
}
}
}

View File

@@ -127,7 +127,7 @@ func testRtPlITConnect(t *testing.T) {
rtplCfgIn.DataDbCfg().Host, rtplCfgIn.DataDbCfg().Port,
rtplCfgIn.DataDbCfg().Name, rtplCfgIn.DataDbCfg().User,
rtplCfgIn.DataDbCfg().Password, rtplCfgIn.GeneralCfg().DBDataEncoding,
config.CgrConfig().CacheCfg(), ratePrfCfgIn.DataDbCfg().Opts)
config.CgrConfig().CacheCfg(), rtplCfgIn.DataDbCfg().Opts)
if err != nil {
log.Fatal(err)
}
@@ -135,7 +135,7 @@ func testRtPlITConnect(t *testing.T) {
rtplCfgOut.DataDbCfg().Host, rtplCfgOut.DataDbCfg().Port,
rtplCfgOut.DataDbCfg().Name, rtplCfgOut.DataDbCfg().User,
rtplCfgOut.DataDbCfg().Password, rtplCfgOut.GeneralCfg().DBDataEncoding,
config.CgrConfig().CacheCfg(), ratePrfCfgOut.DataDbCfg().Opts)
config.CgrConfig().CacheCfg(), rtplCfgOut.DataDbCfg().Opts)
if err != nil {
log.Fatal(err)
}

View File

@@ -1,77 +0,0 @@
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package migrator
import (
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/utils"
)
func (m *Migrator) migrateCurrentTPRateProfiles() (err error) {
tpids, err := m.storDBIn.StorDB().GetTpIds(utils.TBLTPRateProfiles)
if err != nil {
return err
}
for _, tpid := range tpids {
ids, err := m.storDBIn.StorDB().GetTpTableIds(tpid, utils.TBLTPRateProfiles,
utils.TPDistinctIds{"id"}, map[string]string{}, nil)
if err != nil {
return err
}
for _, id := range ids {
rateProfiles, err := m.storDBIn.StorDB().GetTPRateProfiles(tpid, utils.EmptyString, id)
if err != nil {
return err
}
if rateProfiles == nil || m.dryRun {
continue
}
if err := m.storDBOut.StorDB().SetTPRateProfiles(rateProfiles); err != nil {
return err
}
for _, rateProfile := range rateProfiles {
if err := m.storDBIn.StorDB().RemTpData(utils.TBLTPRateProfiles, rateProfile.TPid,
map[string]string{"tenant": rateProfile.Tenant, "id": rateProfile.ID}); err != nil {
return err
}
}
m.stats[utils.TpRateProfiles]++
}
}
return
}
func (m *Migrator) migrateTPRateProfiles() (err error) {
var vrs engine.Versions
current := engine.CurrentStorDBVersions()
if vrs, err = m.getVersions(utils.TpRateProfiles); err != nil {
return err
}
switch vrs[utils.TpRateProfiles] {
case current[utils.TpRateProfiles]:
if m.sameStorDB {
break
}
if err := m.migrateCurrentTPRateProfiles(); err != nil {
return err
}
}
return m.ensureIndexesStorDB(utils.TpRateProfiles)
}

View File

@@ -1,184 +0,0 @@
// +build integration
/*
Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
Copyright (C) ITsysCOM GmbH
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
*/
package migrator
import (
"path"
"reflect"
"testing"
"github.com/cgrates/cgrates/engine"
"github.com/cgrates/cgrates/config"
"github.com/cgrates/cgrates/utils"
)
var (
tpRatePrfPathIn string
tpRatePrfPathOut string
tpRatePrfCfgIn *config.CGRConfig
tpRatePrfCfgOut *config.CGRConfig
tpRatePrfMigrator *Migrator
tpRateProfiles []*utils.TPRateProfile
)
var sTestsTPRatePrfIT = []func(t *testing.T){
testTPRateProfileConnect,
testTPRateProfileFlush,
testTPRateProfilePopulate,
testTpRateProfileMove,
testTpRateProfileCheckData,
}
func TestTPRateProfileIT(t *testing.T) {
for _, tests := range sTestsTPRatePrfIT {
t.Run("TestTPRatePrfIT", tests)
}
tpRatePrfMigrator.Close()
}
func testTPRateProfileConnect(t *testing.T) {
var err error
tpRatePrfPathIn := path.Join(*dataDir, "conf", "samples", "tutmongo")
if tpRatePrfCfgIn, err = config.NewCGRConfigFromPath(tpRatePrfPathIn); err != nil {
t.Error(err)
}
tpRatePrfPathOut := path.Join(*dataDir, "conf", "samples", "tutmysql")
if tpRatePrfCfgOut, err = config.NewCGRConfigFromPath(tpRatePrfPathOut); err != nil {
t.Error(err)
}
storDBIn, err := NewMigratorStorDB(tpRatePrfCfgIn.StorDbCfg().Type,
tpRatePrfCfgIn.StorDbCfg().Host, tpRatePrfCfgIn.StorDbCfg().Port,
tpRatePrfCfgIn.StorDbCfg().Name, tpRatePrfCfgIn.StorDbCfg().User,
tpRatePrfCfgIn.StorDbCfg().Password, tpRatePrfCfgIn.GeneralCfg().DBDataEncoding,
tpRatePrfCfgIn.StorDbCfg().StringIndexedFields, tpRatePrfCfgIn.StorDbCfg().PrefixIndexedFields,
tpRatePrfCfgIn.StorDbCfg().Opts)
if err != nil {
t.Error(err)
}
storDBOut, err := NewMigratorStorDB(tpRatePrfCfgOut.StorDbCfg().Type,
tpRatePrfCfgOut.StorDbCfg().Host, tpRatePrfCfgOut.StorDbCfg().Port,
tpRatePrfCfgOut.StorDbCfg().Name, tpRatePrfCfgOut.StorDbCfg().User,
tpRatePrfCfgOut.StorDbCfg().Password, tpRatePrfCfgOut.GeneralCfg().DBDataEncoding,
tpRatePrfCfgOut.StorDbCfg().StringIndexedFields, tpRatePrfCfgOut.StorDbCfg().PrefixIndexedFields,
tpRatePrfCfgOut.StorDbCfg().Opts)
if err != nil {
t.Error(err)
}
tpRatePrfMigrator, err = NewMigrator(nil, nil, storDBIn, storDBOut,
false, false, false, false)
if err != nil {
t.Error(err)
}
}
func testTPRateProfileFlush(t *testing.T) {
if err := tpRatePrfMigrator.storDBIn.StorDB().Flush(
path.Join(tpRatePrfCfgIn.DataFolderPath, "storage", tpRatePrfCfgIn.StorDbCfg().Type)); err != nil {
t.Error(err)
}
if err := tpRatePrfMigrator.storDBOut.StorDB().Flush(
path.Join(tpRatePrfCfgOut.DataFolderPath, "storage", tpRatePrfCfgOut.StorDbCfg().Type)); err != nil {
t.Error(err)
}
}
func testTPRateProfilePopulate(t *testing.T) {
tpRateProfiles = []*utils.TPRateProfile{
{
TPid: "id_RP1",
Tenant: "cgrates.org",
ID: "RP1",
FilterIDs: []string{"*string:~*req.Subject:1001"},
Weights: ";0",
MinCost: 0.1,
MaxCost: 0.6,
MaxCostStrategy: "*free",
Rates: map[string]*utils.TPRate{
"FIRST_GI": {
ID: "FIRST_GI",
FilterIDs: []string{"*gi:~*req.Usage:0"},
Weights: ";0",
IntervalRates: []*utils.TPIntervalRate{
{
RecurrentFee: 0.12,
Unit: "1m",
Increment: "1m",
},
},
Blocker: false,
},
"SECOND_GI": {
ID: "SECOND_GI",
FilterIDs: []string{"*gi:~*req.Usage:1m"},
Weights: ";10",
IntervalRates: []*utils.TPIntervalRate{
{
RecurrentFee: 0.06,
Unit: "1m",
Increment: "1s",
},
},
Blocker: false,
},
},
},
}
//empty in database
if _, err := tpRatePrfMigrator.storDBIn.StorDB().GetTPRateProfiles(tpRateProfiles[0].TPid,
utils.EmptyString, tpRateProfiles[0].ID); err != utils.ErrNotFound {
t.Error(err)
}
if err := tpRatePrfMigrator.storDBIn.StorDB().SetTPRateProfiles(tpRateProfiles); err != nil {
t.Error("Error when setting TpRateProfile ", err.Error())
}
currentVersion := engine.CurrentStorDBVersions()
err := tpRatePrfMigrator.storDBIn.StorDB().SetVersions(currentVersion, false)
if err != nil {
t.Error("Error when setting version for TpRateProfile ", err.Error())
}
}
func testTpRateProfileMove(t *testing.T) {
err, _ := tpRatePrfMigrator.Migrate([]string{utils.MetaTpRateProfiles})
if err != nil {
t.Error("Error when migrating TpRateProfiles", err.Error())
}
}
func testTpRateProfileCheckData(t *testing.T) {
rcv, err := tpRatePrfMigrator.storDBOut.StorDB().GetTPRateProfiles(tpRateProfiles[0].TPid,
utils.EmptyString, tpRateProfiles[0].ID)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(rcv[0], tpRateProfiles[0]) {
t.Errorf("Expected %+v, received %+v", tpRateProfiles[0], rcv[0])
}
_, err = tpRatePrfMigrator.storDBIn.StorDB().GetTPRateProfiles(tpRateProfiles[0].TPid,
utils.EmptyString, tpRateProfiles[0].ID)
if err != utils.ErrNotFound {
t.Error(err)
}
}