diff --git a/apier/v1/apier.go b/apier/v1/apier.go
index 70bd81a77..7e4709081 100644
--- a/apier/v1/apier.go
+++ b/apier/v1/apier.go
@@ -19,6 +19,7 @@ along with this program. If not, see
package v1
import (
+ "encoding/csv"
"errors"
"fmt"
"io/ioutil"
@@ -1358,3 +1359,61 @@ func (apiv1 *APIerSv1) Ping(ign *utils.CGREvent, reply *string) error {
*reply = utils.Pong
return nil
}
+
+//ExportToFolder export specific items (or all items if items is empty) from DataDB back to CSV
+func (apiV1 *APIerSv1) ExportToFolder(arg *utils.ArgExportToFolder, reply *string) error {
+ // if items is empy we need to export all items
+ if len(arg.Items) == 0 {
+ arg.Items = []string{utils.MetaAttributes, utils.MetaChargers, utils.MetaDispatchers, utils.MetaFilters,
+ utils.MetaResources, utils.MetaStats, utils.MetaSuppliers, utils.MetaThresholds}
+ }
+ for _, item := range arg.Items {
+ switch item {
+ case utils.MetaAttributes:
+ prfx := utils.AttributeProfilePrefix
+ keys, err := apiV1.DataManager.DataDB().GetKeysForPrefix(prfx)
+ if err != nil {
+ return err
+ }
+ // take the tenant + id from key
+ if len(keys) == 0 { // if we don't find items we skip
+ continue
+ }
+ f, err := os.Create(path.Join(arg.Path, utils.AttributesCsv))
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ csvWriter := csv.NewWriter(f)
+ csvWriter.Comma = utils.CSV_SEP
+ //write the header of the file
+ // #Tenant,ID,Contexts,FilterIDs,ActivationInterval,AttributeFilterIDs,Path,Type,Value,Blocker,Weight
+ if err := csvWriter.Write([]string{"#" + utils.Tenant, utils.ID, utils.FilterIDs, utils.ActivationInternal,
+ utils.AttributeFilterIDs, utils.Path, utils.Type, utils.Value, utils.Blocker, utils.Weight}); err != nil {
+ return err
+ }
+ for _, key := range keys {
+ // take tntID from key
+ tntID := strings.SplitN(key[len(prfx):], utils.InInFieldSep, 2)
+ attPrf, err := apiV1.DataManager.GetAttributeProfile(tntID[0], tntID[1],
+ true, false, utils.NonTransactional)
+ if err != nil {
+ return err
+ }
+ for _, model := range engine.APItoModelTPAttribute(
+ engine.AttributeProfileToAPI(attPrf)) {
+ if record, err := engine.CsvDump(model); err != nil {
+ return err
+ } else if err := csvWriter.Write(record); err != nil {
+ return err
+ }
+ }
+
+ }
+ csvWriter.Flush()
+ }
+ }
+ *reply = utils.OK
+ return nil
+}
diff --git a/engine/model_helpers.go b/engine/model_helpers.go
index 1f499cdeb..2588eca25 100644
--- a/engine/model_helpers.go
+++ b/engine/model_helpers.go
@@ -91,7 +91,8 @@ func csvLoad(s interface{}, values []string) (interface{}, error) {
return elem.Interface(), nil
}
-func csvDump(s interface{}) ([]string, error) {
+//CsvDump receive and interface and convert it to a slice of string
+func CsvDump(s interface{}) ([]string, error) {
fieldIndexMap := make(map[string]int)
st := reflect.ValueOf(s)
if st.Kind() == reflect.Ptr {
@@ -2010,6 +2011,8 @@ func APItoSupplierProfile(tpSPP *utils.TPSupplierProfile, timezone string) (spp
type TPAttributes []*TPAttribute
+// create a function for models that return the CSV header
+
func (tps TPAttributes) AsTPAttributes() (result []*utils.TPAttributeProfile) {
mst := make(map[string]*utils.TPAttributeProfile)
filterMap := make(map[string]utils.StringMap)
@@ -2174,16 +2177,17 @@ func APItoAttributeProfile(tpAttr *utils.TPAttributeProfile, timezone string) (a
return attrPrf, nil
}
-func AttributeProfileToAPI(attrPrf *AttributeProfile, tpid string) (tpAttr *utils.TPAttributeProfile) {
+func AttributeProfileToAPI(attrPrf *AttributeProfile) (tpAttr *utils.TPAttributeProfile) {
tpAttr = &utils.TPAttributeProfile{
- TPid: tpid,
- Tenant: attrPrf.Tenant,
- ID: attrPrf.ID,
- FilterIDs: make([]string, len(attrPrf.FilterIDs)),
- Contexts: make([]string, len(attrPrf.Contexts)),
- Attributes: make([]*utils.TPAttribute, len(tpAttr.Attributes)),
- Blocker: attrPrf.Blocker,
- Weight: attrPrf.Weight,
+ TPid: utils.EmptyString,
+ Tenant: attrPrf.Tenant,
+ ID: attrPrf.ID,
+ FilterIDs: make([]string, len(attrPrf.FilterIDs)),
+ Contexts: make([]string, len(attrPrf.Contexts)),
+ Attributes: make([]*utils.TPAttribute, len(attrPrf.Attributes)),
+ ActivationInterval: new(utils.TPActivationInterval),
+ Blocker: attrPrf.Blocker,
+ Weight: attrPrf.Weight,
}
for i, fli := range attrPrf.FilterIDs {
tpAttr.FilterIDs[i] = fli
@@ -2199,9 +2203,13 @@ func AttributeProfileToAPI(attrPrf *AttributeProfile, tpid string) (tpAttr *util
Value: attr.Value.GetRule(),
}
}
- tpAttr.ActivationInterval = &utils.TPActivationInterval{
- ActivationTime: attrPrf.ActivationInterval.ActivationTime.Format(time.RFC3339),
- ExpiryTime: attrPrf.ActivationInterval.ExpiryTime.Format(time.RFC3339),
+ if attrPrf.ActivationInterval != nil {
+ if !attrPrf.ActivationInterval.ActivationTime.IsZero() {
+ tpAttr.ActivationInterval.ActivationTime = attrPrf.ActivationInterval.ActivationTime.Format(time.RFC3339)
+ }
+ if !attrPrf.ActivationInterval.ExpiryTime.IsZero() {
+ tpAttr.ActivationInterval.ExpiryTime = attrPrf.ActivationInterval.ExpiryTime.Format(time.RFC3339)
+ }
}
return
}
diff --git a/engine/model_helpers_test.go b/engine/model_helpers_test.go
index 9cc573868..426885b9d 100644
--- a/engine/model_helpers_test.go
+++ b/engine/model_helpers_test.go
@@ -39,7 +39,7 @@ func TestModelHelperCsvDump(t *testing.T) {
tpd := TpDestination{
Tag: "TEST_DEST",
Prefix: "+492"}
- csv, err := csvDump(tpd)
+ csv, err := CsvDump(tpd)
if err != nil || csv[0] != "TEST_DEST" || csv[1] != "+492" {
t.Errorf("model load failed: %+v", tpd)
}
@@ -59,7 +59,7 @@ func TestTPDestinationAsExportSlice(t *testing.T) {
mdst := APItoModelDestination(tpDst)
var slc [][]string
for _, md := range mdst {
- lc, err := csvDump(md)
+ lc, err := CsvDump(md)
if err != nil {
t.Error("Error dumping to csv: ", err)
}
@@ -109,7 +109,7 @@ func TestTPRateAsExportSlice(t *testing.T) {
ms := APItoModelRate(tpRate)
var slc [][]string
for _, m := range ms {
- lc, err := csvDump(m)
+ lc, err := CsvDump(m)
if err != nil {
t.Error("Error dumping to csv: ", err)
}
@@ -144,7 +144,7 @@ func TestTPDestinationRateAsExportSlice(t *testing.T) {
ms := APItoModelDestinationRate(tpDstRate)
var slc [][]string
for _, m := range ms {
- lc, err := csvDump(m)
+ lc, err := CsvDump(m)
if err != nil {
t.Error("Error dumping to csv: ", err)
}
@@ -172,7 +172,7 @@ func TestApierTPTimingAsExportSlice(t *testing.T) {
ms := APItoModelTiming(tpTiming)
var slc [][]string
- lc, err := csvDump(ms)
+ lc, err := CsvDump(ms)
if err != nil {
t.Error("Error dumping to csv: ", err)
}
@@ -205,7 +205,7 @@ func TestTPRatingPlanAsExportSlice(t *testing.T) {
ms := APItoModelRatingPlan(tpRpln)
var slc [][]string
for _, m := range ms {
- lc, err := csvDump(m)
+ lc, err := CsvDump(m)
if err != nil {
t.Error("Error dumping to csv: ", err)
}
@@ -242,7 +242,7 @@ func TestTPRatingProfileAsExportSlice(t *testing.T) {
ms := APItoModelRatingProfile(tpRpf)
var slc [][]string
for _, m := range ms {
- lc, err := csvDump(m)
+ lc, err := CsvDump(m)
if err != nil {
t.Error("Error dumping to csv: ", err)
}
@@ -293,7 +293,7 @@ func TestTPActionsAsExportSlice(t *testing.T) {
ms := APItoModelAction(tpActs)
var slc [][]string
for _, m := range ms {
- lc, err := csvDump(m)
+ lc, err := CsvDump(m)
if err != nil {
t.Error("Error dumping to csv: ", err)
}
@@ -329,7 +329,7 @@ func TestTPSharedGroupsAsExportSlice(t *testing.T) {
ms := APItoModelSharedGroup(tpSGs)
var slc [][]string
for _, m := range ms {
- lc, err := csvDump(m)
+ lc, err := CsvDump(m)
if err != nil {
t.Error("Error dumping to csv: ", err)
}
@@ -362,7 +362,7 @@ func TestTPActionTriggersAsExportSlice(t *testing.T) {
ms := APItoModelActionPlan(ap)
var slc [][]string
for _, m := range ms {
- lc, err := csvDump(m)
+ lc, err := CsvDump(m)
if err != nil {
t.Error("Error dumping to csv: ", err)
}
@@ -427,7 +427,7 @@ func TestTPActionPlanAsExportSlice(t *testing.T) {
ms := APItoModelActionTrigger(at)
var slc [][]string
for _, m := range ms {
- lc, err := csvDump(m)
+ lc, err := CsvDump(m)
if err != nil {
t.Error("Error dumping to csv: ", err)
}
@@ -452,7 +452,7 @@ func TestTPAccountActionsAsExportSlice(t *testing.T) {
}
ms := APItoModelAccountAction(aa)
var slc [][]string
- lc, err := csvDump(*ms)
+ lc, err := CsvDump(*ms)
if err != nil {
t.Error("Error dumping to csv: ", err)
}
@@ -1240,6 +1240,94 @@ func TestAPItoAttributeProfile(t *testing.T) {
}
}
+func TestAttributeProfileToAPI(t *testing.T) {
+ exp := &utils.TPAttributeProfile{
+ TPid: utils.EmptyString,
+ Tenant: "cgrates.org",
+ ID: "ALS1",
+ Contexts: []string{"con1"},
+ FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"},
+ ActivationInterval: &utils.TPActivationInterval{
+ ActivationTime: "2014-07-14T14:35:00Z",
+ ExpiryTime: "",
+ },
+ Attributes: []*utils.TPAttribute{
+ &utils.TPAttribute{
+ Path: utils.MetaReq + utils.NestingSep + "FL1",
+ Value: "Al1",
+ },
+ },
+ Weight: 20,
+ }
+ attr := &AttributeProfile{
+ Tenant: "cgrates.org",
+ ID: "ALS1",
+ Contexts: []string{"con1"},
+ FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"},
+ ActivationInterval: &utils.ActivationInterval{
+ ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC),
+ },
+ Attributes: []*Attribute{
+ &Attribute{
+ Path: utils.MetaReq + utils.NestingSep + "FL1",
+ Value: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP),
+ },
+ },
+ Weight: 20,
+ }
+ if rcv := AttributeProfileToAPI(attr); !reflect.DeepEqual(exp, rcv) {
+ t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(exp), utils.ToJSON(rcv))
+ }
+}
+
+func TestAttributeProfileToAPI2(t *testing.T) {
+ exp := &utils.TPAttributeProfile{
+ TPid: utils.EmptyString,
+ Tenant: "cgrates.org",
+ ID: "ALS1",
+ Contexts: []string{"con1"},
+ FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"},
+ ActivationInterval: &utils.TPActivationInterval{
+ ActivationTime: "2014-07-14T14:35:00Z",
+ ExpiryTime: "",
+ },
+ Attributes: []*utils.TPAttribute{
+ &utils.TPAttribute{
+ Path: utils.MetaReq + utils.NestingSep + "FL1",
+ Value: "Al1",
+ },
+ &utils.TPAttribute{
+ Path: utils.MetaReq + utils.NestingSep + "Test",
+ Value: "~*req.Account",
+ },
+ },
+ Weight: 20,
+ }
+ attr := &AttributeProfile{
+ Tenant: "cgrates.org",
+ ID: "ALS1",
+ Contexts: []string{"con1"},
+ FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"},
+ ActivationInterval: &utils.ActivationInterval{
+ ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC),
+ },
+ Attributes: []*Attribute{
+ &Attribute{
+ Path: utils.MetaReq + utils.NestingSep + "FL1",
+ Value: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP),
+ },
+ &Attribute{
+ Path: utils.MetaReq + utils.NestingSep + "Test",
+ Value: config.NewRSRParsersMustCompile("~*req.Account", true, utils.INFIELD_SEP),
+ },
+ },
+ Weight: 20,
+ }
+ if rcv := AttributeProfileToAPI(attr); !reflect.DeepEqual(exp, rcv) {
+ t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(exp), utils.ToJSON(rcv))
+ }
+}
+
func TestAPItoModelTPAttribute(t *testing.T) {
tpAlsPrf := &utils.TPAttributeProfile{
TPid: "TP1",
diff --git a/engine/tpexporter.go b/engine/tpexporter.go
index d2298ffad..6adb5b66f 100644
--- a/engine/tpexporter.go
+++ b/engine/tpexporter.go
@@ -367,7 +367,7 @@ func (self *TPExporter) writeOut(fileName string, tpData []interface{}) error {
writerOut = utils.NewCgrIORecordWriter(fWriter)
}
for _, tpItem := range tpData {
- record, err := csvDump(tpItem)
+ record, err := CsvDump(tpItem)
if err != nil {
return err
}
diff --git a/general_tests/export_it_test.go b/general_tests/export_it_test.go
new file mode 100644
index 000000000..a76d32449
--- /dev/null
+++ b/general_tests/export_it_test.go
@@ -0,0 +1,129 @@
+// +build integration
+
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see
+*/
+package general_tests
+
+import (
+ "net/rpc"
+ "path"
+ "testing"
+
+ "github.com/cgrates/cgrates/config"
+ "github.com/cgrates/cgrates/engine"
+ "github.com/cgrates/cgrates/utils"
+)
+
+var (
+ expCfgDir string
+ expCfgPath string
+ expCfg *config.CGRConfig
+ expRpc *rpc.Client
+
+ sTestsExp = []func(t *testing.T){
+ testExpLoadConfig,
+ testExpResetDataDB,
+ testExpResetStorDb,
+ testExpStartEngine,
+ testExpRPCConn,
+ testExpLoadTPFromFolder,
+ testExpAttribute,
+ testExpStopCgrEngine,
+ }
+)
+
+func TestExport(t *testing.T) {
+ switch *dbType {
+ case utils.MetaInternal:
+ expCfgDir = "tutinternal"
+ case utils.MetaMySQL:
+ expCfgDir = "tutmysql"
+ case utils.MetaMongo:
+ expCfgDir = "tutmongo"
+ case utils.MetaPostgres:
+ t.SkipNow()
+ default:
+ t.Fatal("Unknown Database type")
+ }
+
+ for _, stest := range sTestsExp {
+ t.Run(expCfgDir, stest)
+ }
+}
+
+func testExpLoadConfig(t *testing.T) {
+ expCfgPath = path.Join(*dataDir, "conf", "samples", expCfgDir)
+ if expCfg, err = config.NewCGRConfigFromPath(expCfgPath); err != nil {
+ t.Error(err)
+ }
+}
+
+func testExpResetDataDB(t *testing.T) {
+ if err := engine.InitDataDb(expCfg); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func testExpResetStorDb(t *testing.T) {
+ if err := engine.InitStorDb(expCfg); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func testExpStartEngine(t *testing.T) {
+ if _, err := engine.StopStartEngine(expCfgPath, *waitRater); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func testExpRPCConn(t *testing.T) {
+ var err error
+ expRpc, err = newRPCClient(expCfg.ListenCfg())
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func testExpLoadTPFromFolder(t *testing.T) {
+ var reply string
+ attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tariffplans", "tutorial")}
+ if err := expRpc.Call(utils.APIerSv1LoadTariffPlanFromFolder, attrs, &reply); err != nil {
+ t.Error(err)
+ } else if reply != utils.OK {
+ t.Error(reply)
+ }
+}
+
+func testExpAttribute(t *testing.T) {
+ var reply string
+ arg := &utils.ArgExportToFolder{
+ Path: "/tmp",
+ Items: []string{utils.MetaAttributes},
+ }
+ if err := expRpc.Call(utils.APIerSv1ExportToFolder, arg, &reply); err != nil {
+ t.Error(err)
+ } else if reply != utils.OK {
+ t.Error(reply)
+ }
+}
+
+func testExpStopCgrEngine(t *testing.T) {
+ if err := engine.KillEngine(100); err != nil {
+ t.Error(err)
+ }
+}
diff --git a/utils/apitpdata.go b/utils/apitpdata.go
index 8c2cb3cc1..d508acb65 100755
--- a/utils/apitpdata.go
+++ b/utils/apitpdata.go
@@ -1418,3 +1418,9 @@ type GetMaxSessionTimeOnAccountsArgs struct {
Usage time.Duration
AccountIDs []string
}
+
+type ArgExportToFolder struct {
+ Tenant string
+ Path string
+ Items []string
+}
diff --git a/utils/consts.go b/utils/consts.go
index 76fb03056..e5c9699eb 100755
--- a/utils/consts.go
+++ b/utils/consts.go
@@ -534,6 +534,8 @@ const (
MetaCGRAReq = "*cgrareq"
CGR_ACD = "cgr_acd"
FilterIDs = "FilterIDs"
+ ActivationInternal = "ActivationInterval"
+ AttributeFilterIDs = "AttributeFilterIDs"
FieldName = "FieldName"
Path = "Path"
MetaRound = "*round"
@@ -995,6 +997,7 @@ const (
APIerSv1RemoveDispatcherHost = "APIerSv1.RemoveDispatcherHost"
APIerSv1GetEventCost = "APIerSv1.GetEventCost"
APIerSv1LoadTariffPlanFromFolder = "APIerSv1.LoadTariffPlanFromFolder"
+ APIerSv1ExportToFolder = "APIerSv1.ExportToFolder"
APIerSv1GetCost = "APIerSv1.GetCost"
APIerSv1SetBalance = "APIerSv1.SetBalance"
APIerSv1GetFilter = "APIerSv1.GetFilter"