Add a PoC for ExportToFile

apier/v1/apier.go

@@ -19,6 +19,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package v1
 
 import (
+	"encoding/csv"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -1358,3 +1359,61 @@ func (apiv1 *APIerSv1) Ping(ign *utils.CGREvent, reply *string) error {
 	*reply = utils.Pong
 	return nil
 }
+
+//ExportToFolder export specific items (or all items if items is empty) from DataDB back to CSV
+func (apiV1 *APIerSv1) ExportToFolder(arg *utils.ArgExportToFolder, reply *string) error {
+	// if items is empy we need to export all items
+	if len(arg.Items) == 0 {
+		arg.Items = []string{utils.MetaAttributes, utils.MetaChargers, utils.MetaDispatchers, utils.MetaFilters,
+			utils.MetaResources, utils.MetaStats, utils.MetaSuppliers, utils.MetaThresholds}
+	}
+	for _, item := range arg.Items {
+		switch item {
+		case utils.MetaAttributes:
+			prfx := utils.AttributeProfilePrefix
+			keys, err := apiV1.DataManager.DataDB().GetKeysForPrefix(prfx)
+			if err != nil {
+				return err
+			}
+			// take the tenant + id from key
+			if len(keys) == 0 { // if we don't find items we skip
+				continue
+			}
+			f, err := os.Create(path.Join(arg.Path, utils.AttributesCsv))
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+
+			csvWriter := csv.NewWriter(f)
+			csvWriter.Comma = utils.CSV_SEP
+			//write the header of the file
+			// #Tenant,ID,Contexts,FilterIDs,ActivationInterval,AttributeFilterIDs,Path,Type,Value,Blocker,Weight
+			if err := csvWriter.Write([]string{"#" + utils.Tenant, utils.ID, utils.FilterIDs, utils.ActivationInternal,
+				utils.AttributeFilterIDs, utils.Path, utils.Type, utils.Value, utils.Blocker, utils.Weight}); err != nil {
+				return err
+			}
+			for _, key := range keys {
+				// take tntID from key
+				tntID := strings.SplitN(key[len(prfx):], utils.InInFieldSep, 2)
+				attPrf, err := apiV1.DataManager.GetAttributeProfile(tntID[0], tntID[1],
+					true, false, utils.NonTransactional)
+				if err != nil {
+					return err
+				}
+				for _, model := range engine.APItoModelTPAttribute(
+					engine.AttributeProfileToAPI(attPrf)) {
+					if record, err := engine.CsvDump(model); err != nil {
+						return err
+					} else if err := csvWriter.Write(record); err != nil {
+						return err
+					}
+				}
+
+			}
+			csvWriter.Flush()
+		}
+	}
+	*reply = utils.OK
+	return nil
+}

engine/model_helpers.go

@@ -91,7 +91,8 @@ func csvLoad(s interface{}, values []string) (interface{}, error) {
 	return elem.Interface(), nil
 }
 
-func csvDump(s interface{}) ([]string, error) {
+//CsvDump receive and interface and convert it to a slice of string
+func CsvDump(s interface{}) ([]string, error) {
 	fieldIndexMap := make(map[string]int)
 	st := reflect.ValueOf(s)
 	if st.Kind() == reflect.Ptr {
@@ -2010,6 +2011,8 @@ func APItoSupplierProfile(tpSPP *utils.TPSupplierProfile, timezone string) (spp
 
 type TPAttributes []*TPAttribute
 
+// create a function for models that return the CSV header
+
 func (tps TPAttributes) AsTPAttributes() (result []*utils.TPAttributeProfile) {
 	mst := make(map[string]*utils.TPAttributeProfile)
 	filterMap := make(map[string]utils.StringMap)
@@ -2174,16 +2177,17 @@ func APItoAttributeProfile(tpAttr *utils.TPAttributeProfile, timezone string) (a
 	return attrPrf, nil
 }
 
-func AttributeProfileToAPI(attrPrf *AttributeProfile, tpid string) (tpAttr *utils.TPAttributeProfile) {
+func AttributeProfileToAPI(attrPrf *AttributeProfile) (tpAttr *utils.TPAttributeProfile) {
 	tpAttr = &utils.TPAttributeProfile{
-		TPid:       tpid,
-		Tenant:     attrPrf.Tenant,
-		ID:         attrPrf.ID,
-		FilterIDs:  make([]string, len(attrPrf.FilterIDs)),
-		Contexts:   make([]string, len(attrPrf.Contexts)),
-		Attributes: make([]*utils.TPAttribute, len(tpAttr.Attributes)),
-		Blocker:    attrPrf.Blocker,
-		Weight:     attrPrf.Weight,
+		TPid:               utils.EmptyString,
+		Tenant:             attrPrf.Tenant,
+		ID:                 attrPrf.ID,
+		FilterIDs:          make([]string, len(attrPrf.FilterIDs)),
+		Contexts:           make([]string, len(attrPrf.Contexts)),
+		Attributes:         make([]*utils.TPAttribute, len(attrPrf.Attributes)),
+		ActivationInterval: new(utils.TPActivationInterval),
+		Blocker:            attrPrf.Blocker,
+		Weight:             attrPrf.Weight,
 	}
 	for i, fli := range attrPrf.FilterIDs {
 		tpAttr.FilterIDs[i] = fli
@@ -2199,9 +2203,13 @@ func AttributeProfileToAPI(attrPrf *AttributeProfile, tpid string) (tpAttr *util
 			Value:     attr.Value.GetRule(),
 		}
 	}
-	tpAttr.ActivationInterval = &utils.TPActivationInterval{
-		ActivationTime: attrPrf.ActivationInterval.ActivationTime.Format(time.RFC3339),
-		ExpiryTime:     attrPrf.ActivationInterval.ExpiryTime.Format(time.RFC3339),
+	if attrPrf.ActivationInterval != nil {
+		if !attrPrf.ActivationInterval.ActivationTime.IsZero() {
+			tpAttr.ActivationInterval.ActivationTime = attrPrf.ActivationInterval.ActivationTime.Format(time.RFC3339)
+		}
+		if !attrPrf.ActivationInterval.ExpiryTime.IsZero() {
+			tpAttr.ActivationInterval.ExpiryTime = attrPrf.ActivationInterval.ExpiryTime.Format(time.RFC3339)
+		}
 	}
 	return
 }

engine/model_helpers_test.go

@@ -39,7 +39,7 @@ func TestModelHelperCsvDump(t *testing.T) {
 	tpd := TpDestination{
 		Tag:    "TEST_DEST",
 		Prefix: "+492"}
-	csv, err := csvDump(tpd)
+	csv, err := CsvDump(tpd)
 	if err != nil || csv[0] != "TEST_DEST" || csv[1] != "+492" {
 		t.Errorf("model load failed: %+v", tpd)
 	}
@@ -59,7 +59,7 @@ func TestTPDestinationAsExportSlice(t *testing.T) {
 	mdst := APItoModelDestination(tpDst)
 	var slc [][]string
 	for _, md := range mdst {
-		lc, err := csvDump(md)
+		lc, err := CsvDump(md)
 		if err != nil {
 			t.Error("Error dumping to csv: ", err)
 		}
@@ -109,7 +109,7 @@ func TestTPRateAsExportSlice(t *testing.T) {
 	ms := APItoModelRate(tpRate)
 	var slc [][]string
 	for _, m := range ms {
-		lc, err := csvDump(m)
+		lc, err := CsvDump(m)
 		if err != nil {
 			t.Error("Error dumping to csv: ", err)
 		}
@@ -144,7 +144,7 @@ func TestTPDestinationRateAsExportSlice(t *testing.T) {
 	ms := APItoModelDestinationRate(tpDstRate)
 	var slc [][]string
 	for _, m := range ms {
-		lc, err := csvDump(m)
+		lc, err := CsvDump(m)
 		if err != nil {
 			t.Error("Error dumping to csv: ", err)
 		}
@@ -172,7 +172,7 @@ func TestApierTPTimingAsExportSlice(t *testing.T) {
 	ms := APItoModelTiming(tpTiming)
 	var slc [][]string
 
-	lc, err := csvDump(ms)
+	lc, err := CsvDump(ms)
 	if err != nil {
 		t.Error("Error dumping to csv: ", err)
 	}
@@ -205,7 +205,7 @@ func TestTPRatingPlanAsExportSlice(t *testing.T) {
 	ms := APItoModelRatingPlan(tpRpln)
 	var slc [][]string
 	for _, m := range ms {
-		lc, err := csvDump(m)
+		lc, err := CsvDump(m)
 		if err != nil {
 			t.Error("Error dumping to csv: ", err)
 		}
@@ -242,7 +242,7 @@ func TestTPRatingProfileAsExportSlice(t *testing.T) {
 	ms := APItoModelRatingProfile(tpRpf)
 	var slc [][]string
 	for _, m := range ms {
-		lc, err := csvDump(m)
+		lc, err := CsvDump(m)
 		if err != nil {
 			t.Error("Error dumping to csv: ", err)
 		}
@@ -293,7 +293,7 @@ func TestTPActionsAsExportSlice(t *testing.T) {
 	ms := APItoModelAction(tpActs)
 	var slc [][]string
 	for _, m := range ms {
-		lc, err := csvDump(m)
+		lc, err := CsvDump(m)
 		if err != nil {
 			t.Error("Error dumping to csv: ", err)
 		}
@@ -329,7 +329,7 @@ func TestTPSharedGroupsAsExportSlice(t *testing.T) {
 	ms := APItoModelSharedGroup(tpSGs)
 	var slc [][]string
 	for _, m := range ms {
-		lc, err := csvDump(m)
+		lc, err := CsvDump(m)
 		if err != nil {
 			t.Error("Error dumping to csv: ", err)
 		}
@@ -362,7 +362,7 @@ func TestTPActionTriggersAsExportSlice(t *testing.T) {
 	ms := APItoModelActionPlan(ap)
 	var slc [][]string
 	for _, m := range ms {
-		lc, err := csvDump(m)
+		lc, err := CsvDump(m)
 		if err != nil {
 			t.Error("Error dumping to csv: ", err)
 		}
@@ -427,7 +427,7 @@ func TestTPActionPlanAsExportSlice(t *testing.T) {
 	ms := APItoModelActionTrigger(at)
 	var slc [][]string
 	for _, m := range ms {
-		lc, err := csvDump(m)
+		lc, err := CsvDump(m)
 		if err != nil {
 			t.Error("Error dumping to csv: ", err)
 		}
@@ -452,7 +452,7 @@ func TestTPAccountActionsAsExportSlice(t *testing.T) {
 	}
 	ms := APItoModelAccountAction(aa)
 	var slc [][]string
-	lc, err := csvDump(*ms)
+	lc, err := CsvDump(*ms)
 	if err != nil {
 		t.Error("Error dumping to csv: ", err)
 	}
@@ -1240,6 +1240,94 @@ func TestAPItoAttributeProfile(t *testing.T) {
 	}
 }
 
+func TestAttributeProfileToAPI(t *testing.T) {
+	exp := &utils.TPAttributeProfile{
+		TPid:      utils.EmptyString,
+		Tenant:    "cgrates.org",
+		ID:        "ALS1",
+		Contexts:  []string{"con1"},
+		FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"},
+		ActivationInterval: &utils.TPActivationInterval{
+			ActivationTime: "2014-07-14T14:35:00Z",
+			ExpiryTime:     "",
+		},
+		Attributes: []*utils.TPAttribute{
+			&utils.TPAttribute{
+				Path:  utils.MetaReq + utils.NestingSep + "FL1",
+				Value: "Al1",
+			},
+		},
+		Weight: 20,
+	}
+	attr := &AttributeProfile{
+		Tenant:    "cgrates.org",
+		ID:        "ALS1",
+		Contexts:  []string{"con1"},
+		FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"},
+		ActivationInterval: &utils.ActivationInterval{
+			ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC),
+		},
+		Attributes: []*Attribute{
+			&Attribute{
+				Path:  utils.MetaReq + utils.NestingSep + "FL1",
+				Value: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP),
+			},
+		},
+		Weight: 20,
+	}
+	if rcv := AttributeProfileToAPI(attr); !reflect.DeepEqual(exp, rcv) {
+		t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(exp), utils.ToJSON(rcv))
+	}
+}
+
+func TestAttributeProfileToAPI2(t *testing.T) {
+	exp := &utils.TPAttributeProfile{
+		TPid:      utils.EmptyString,
+		Tenant:    "cgrates.org",
+		ID:        "ALS1",
+		Contexts:  []string{"con1"},
+		FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"},
+		ActivationInterval: &utils.TPActivationInterval{
+			ActivationTime: "2014-07-14T14:35:00Z",
+			ExpiryTime:     "",
+		},
+		Attributes: []*utils.TPAttribute{
+			&utils.TPAttribute{
+				Path:  utils.MetaReq + utils.NestingSep + "FL1",
+				Value: "Al1",
+			},
+			&utils.TPAttribute{
+				Path:  utils.MetaReq + utils.NestingSep + "Test",
+				Value: "~*req.Account",
+			},
+		},
+		Weight: 20,
+	}
+	attr := &AttributeProfile{
+		Tenant:    "cgrates.org",
+		ID:        "ALS1",
+		Contexts:  []string{"con1"},
+		FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"},
+		ActivationInterval: &utils.ActivationInterval{
+			ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC),
+		},
+		Attributes: []*Attribute{
+			&Attribute{
+				Path:  utils.MetaReq + utils.NestingSep + "FL1",
+				Value: config.NewRSRParsersMustCompile("Al1", true, utils.INFIELD_SEP),
+			},
+			&Attribute{
+				Path:  utils.MetaReq + utils.NestingSep + "Test",
+				Value: config.NewRSRParsersMustCompile("~*req.Account", true, utils.INFIELD_SEP),
+			},
+		},
+		Weight: 20,
+	}
+	if rcv := AttributeProfileToAPI(attr); !reflect.DeepEqual(exp, rcv) {
+		t.Errorf("Expecting : %+v, received: %+v", utils.ToJSON(exp), utils.ToJSON(rcv))
+	}
+}
+
 func TestAPItoModelTPAttribute(t *testing.T) {
 	tpAlsPrf := &utils.TPAttributeProfile{
 		TPid:      "TP1",

engine/tpexporter.go

@@ -367,7 +367,7 @@ func (self *TPExporter) writeOut(fileName string, tpData []interface{}) error {
 		writerOut = utils.NewCgrIORecordWriter(fWriter)
 	}
 	for _, tpItem := range tpData {
-		record, err := csvDump(tpItem)
+		record, err := CsvDump(tpItem)
 		if err != nil {
 			return err
 		}

general_tests/export_it_test.go

@@ -0,0 +1,129 @@
+// +build integration
+
+/*
+Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+package general_tests
+
+import (
+	"net/rpc"
+	"path"
+	"testing"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var (
+	expCfgDir  string
+	expCfgPath string
+	expCfg     *config.CGRConfig
+	expRpc     *rpc.Client
+
+	sTestsExp = []func(t *testing.T){
+		testExpLoadConfig,
+		testExpResetDataDB,
+		testExpResetStorDb,
+		testExpStartEngine,
+		testExpRPCConn,
+		testExpLoadTPFromFolder,
+		testExpAttribute,
+		testExpStopCgrEngine,
+	}
+)
+
+func TestExport(t *testing.T) {
+	switch *dbType {
+	case utils.MetaInternal:
+		expCfgDir = "tutinternal"
+	case utils.MetaMySQL:
+		expCfgDir = "tutmysql"
+	case utils.MetaMongo:
+		expCfgDir = "tutmongo"
+	case utils.MetaPostgres:
+		t.SkipNow()
+	default:
+		t.Fatal("Unknown Database type")
+	}
+
+	for _, stest := range sTestsExp {
+		t.Run(expCfgDir, stest)
+	}
+}
+
+func testExpLoadConfig(t *testing.T) {
+	expCfgPath = path.Join(*dataDir, "conf", "samples", expCfgDir)
+	if expCfg, err = config.NewCGRConfigFromPath(expCfgPath); err != nil {
+		t.Error(err)
+	}
+}
+
+func testExpResetDataDB(t *testing.T) {
+	if err := engine.InitDataDb(expCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testExpResetStorDb(t *testing.T) {
+	if err := engine.InitStorDb(expCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testExpStartEngine(t *testing.T) {
+	if _, err := engine.StopStartEngine(expCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testExpRPCConn(t *testing.T) {
+	var err error
+	expRpc, err = newRPCClient(expCfg.ListenCfg())
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testExpLoadTPFromFolder(t *testing.T) {
+	var reply string
+	attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tariffplans", "tutorial")}
+	if err := expRpc.Call(utils.APIerSv1LoadTariffPlanFromFolder, attrs, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error(reply)
+	}
+}
+
+func testExpAttribute(t *testing.T) {
+	var reply string
+	arg := &utils.ArgExportToFolder{
+		Path:  "/tmp",
+		Items: []string{utils.MetaAttributes},
+	}
+	if err := expRpc.Call(utils.APIerSv1ExportToFolder, arg, &reply); err != nil {
+		t.Error(err)
+	} else if reply != utils.OK {
+		t.Error(reply)
+	}
+}
+
+func testExpStopCgrEngine(t *testing.T) {
+	if err := engine.KillEngine(100); err != nil {
+		t.Error(err)
+	}
+}

utils/apitpdata.go

@@ -1418,3 +1418,9 @@ type GetMaxSessionTimeOnAccountsArgs struct {
 	Usage       time.Duration
 	AccountIDs  []string
 }
+
+type ArgExportToFolder struct {
+	Tenant string
+	Path   string
+	Items  []string
+}

utils/consts.go

@@ -534,6 +534,8 @@ const (
 	MetaCGRAReq                 = "*cgrareq"
 	CGR_ACD                     = "cgr_acd"
 	FilterIDs                   = "FilterIDs"
+	ActivationInternal          = "ActivationInterval"
+	AttributeFilterIDs          = "AttributeFilterIDs"
 	FieldName                   = "FieldName"
 	Path                        = "Path"
 	MetaRound                   = "*round"
@@ -995,6 +997,7 @@ const (
 	APIerSv1RemoveDispatcherHost        = "APIerSv1.RemoveDispatcherHost"
 	APIerSv1GetEventCost                = "APIerSv1.GetEventCost"
 	APIerSv1LoadTariffPlanFromFolder    = "APIerSv1.LoadTariffPlanFromFolder"
+	APIerSv1ExportToFolder              = "APIerSv1.ExportToFolder"
 	APIerSv1GetCost                     = "APIerSv1.GetCost"
 	APIerSv1SetBalance                  = "APIerSv1.SetBalance"
 	APIerSv1GetFilter                   = "APIerSv1.GetFilter"