Updated cgr-loader to accept URLs as path to data

cmd/cgr-loader/cgr-loader.go

@@ -19,6 +19,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"log"
@@ -32,10 +33,8 @@ import (
 )
 
 var (
-	err    error
-	dm     *engine.DataManager
-	storDb engine.LoadStorage
-	loader engine.LoadReader
+	dataDB engine.DataDB
+	storDB engine.LoadStorage
 
 	cgrLoaderFlags = flag.NewFlagSet("cgr-loader", flag.ContinueOnError)
 	dfltCfg        = config.CgrConfig()
@@ -106,8 +105,9 @@ var (
 func loadConfig() (ldrCfg *config.CGRConfig) {
 	ldrCfg = config.CgrConfig()
 	if *cfgPath != "" {
+		var err error
 		if ldrCfg, err = config.NewCGRConfigFromPath(*cfgPath); err != nil {
-			log.Fatalf("Error loading config file %s", err.Error())
+			log.Fatalf("Error loading config file %s", err)
 		}
 		config.SetCgrConfig(ldrCfg)
 	}
@@ -226,41 +226,80 @@ func loadConfig() (ldrCfg *config.CGRConfig) {
 	if *disableReverse != dfltCfg.LoaderCgrCfg().DisableReverse {
 		ldrCfg.LoaderCgrCfg().DisableReverse = *disableReverse
 	}
+
+	if *cachingArg != utils.EmptyString {
+		ldrCfg.GeneralCfg().DefaultCaching = *cachingArg
+	}
+	return
+}
+
+func importData(cfg *config.CGRConfig) (err error) {
+	if cfg.LoaderCgrCfg().TpID == "" {
+		return errors.New("TPid required")
+	}
+	if *flushStorDB {
+		if err = storDB.RemTpData("", cfg.LoaderCgrCfg().TpID, map[string]string{}); err != nil {
+			return err
+		}
+	}
+	csvImporter := engine.TPCSVImporter{
+		TPid:     cfg.LoaderCgrCfg().TpID,
+		StorDb:   storDB,
+		DirPath:  *dataPath,
+		Sep:      cfg.LoaderCgrCfg().FieldSeparator,
+		Verbose:  *verbose,
+		ImportId: *importID,
+	}
+	return csvImporter.Run()
+}
+
+func getLoader(cfg *config.CGRConfig) (loader engine.LoadReader, err error) {
+	if *fromStorDB { // Load Tariff Plan from storDb into dataDb
+		loader = storDB
+		return
+	}
+	if gprefix := utils.MetaGoogleAPI + utils.CONCATENATED_KEY_SEP; strings.HasPrefix(*dataPath, gprefix) { // Default load from csv files to dataDb
+		return engine.NewGoogleCSVStorage(cfg.LoaderCgrCfg().FieldSeparator, strings.TrimPrefix(*dataPath, gprefix), *cfgPath)
+	}
+	if !utils.IsURL(*dataPath) {
+		loader = engine.NewFileCSVStorage(cfg.LoaderCgrCfg().FieldSeparator, *dataPath)
+		return
+	}
+	loader = engine.NewURLCSVStorage(cfg.LoaderCgrCfg().FieldSeparator, *dataPath)
 	return
 }
 
 func main() {
-	if err := cgrLoaderFlags.Parse(os.Args[1:]); err != nil {
-		return
+	var err error
+	if err = cgrLoaderFlags.Parse(os.Args[1:]); err != nil {
+		log.Fatal(err)
 	}
 	if *version {
-		if rcv, err := utils.GetCGRVersion(); err != nil {
-			fmt.Println(err)
-		} else {
-			fmt.Println(rcv)
+		var version string
+		if version, err = utils.GetCGRVersion(); err != nil {
+			log.Fatal(err)
 		}
+		fmt.Println(version)
 		return
 	}
 
 	ldrCfg := loadConfig()
 	// we initialize connManager here with nil for InternalChannels
-	cM := engine.NewConnManager(ldrCfg, nil)
+	engine.NewConnManager(ldrCfg, nil)
 
 	if !*toStorDB {
-		d, err := engine.NewDataDBConn(ldrCfg.DataDbCfg().DataDbType,
+		if dataDB, err = engine.NewDataDBConn(ldrCfg.DataDbCfg().DataDbType,
 			ldrCfg.DataDbCfg().DataDbHost, ldrCfg.DataDbCfg().DataDbPort,
 			ldrCfg.DataDbCfg().DataDbName, ldrCfg.DataDbCfg().DataDbUser,
 			ldrCfg.DataDbCfg().DataDbPass, ldrCfg.GeneralCfg().DBDataEncoding,
-			ldrCfg.DataDbCfg().DataDbSentinelName, ldrCfg.DataDbCfg().Items)
-		if err != nil {
+			ldrCfg.DataDbCfg().DataDbSentinelName, ldrCfg.DataDbCfg().Items); err != nil {
 			log.Fatalf("Coud not open dataDB connection: %s", err.Error())
 		}
-		dm = engine.NewDataManager(d, config.CgrConfig().CacheCfg(), cM)
-		defer dm.DataDB().Close()
+		defer dataDB.Close()
 	}
 
 	if *fromStorDB || *toStorDB {
-		if storDb, err = engine.NewStorDBConn(ldrCfg.StorDbCfg().Type,
+		if storDB, err = engine.NewStorDBConn(ldrCfg.StorDbCfg().Type,
 			ldrCfg.StorDbCfg().Host, ldrCfg.StorDbCfg().Port,
 			ldrCfg.StorDbCfg().Name, ldrCfg.StorDbCfg().User,
 			ldrCfg.StorDbCfg().Password, ldrCfg.GeneralCfg().DBDataEncoding, ldrCfg.StorDbCfg().SSLMode,
@@ -269,46 +308,24 @@ func main() {
 			ldrCfg.StorDbCfg().PrefixIndexedFields, ldrCfg.StorDbCfg().Items); err != nil {
 			log.Fatalf("Coud not open storDB connection: %s", err.Error())
 		}
-		defer storDb.Close()
+		defer storDB.Close()
 	}
 
 	if !*dryRun && *toStorDB { // Import files from a directory into storDb
-		if ldrCfg.LoaderCgrCfg().TpID == "" {
-			log.Fatal("TPid required.")
-		}
-		if *flushStorDB {
-			if err = storDb.RemTpData("", ldrCfg.LoaderCgrCfg().TpID, map[string]string{}); err != nil {
-				log.Fatal(err)
-			}
-		}
-		csvImporter := engine.TPCSVImporter{
-			TPid:     ldrCfg.LoaderCgrCfg().TpID,
-			StorDb:   storDb,
-			DirPath:  *dataPath,
-			Sep:      ldrCfg.LoaderCgrCfg().FieldSeparator,
-			Verbose:  *verbose,
-			ImportId: *importID,
-		}
-		if errImport := csvImporter.Run(); errImport != nil {
-			log.Fatal(errImport)
+		if err = importData(ldrCfg); err != nil {
+			log.Fatal(err)
 		}
 		return
 	}
-
-	if *fromStorDB { // Load Tariff Plan from storDb into dataDb
-		loader = storDb
-	} else if gprefix := utils.MetaGoogleAPI + utils.CONCATENATED_KEY_SEP; strings.HasPrefix(*dataPath, gprefix) { // Default load from csv files to dataDb
-		loader, err = engine.NewGoogleCSVStorage(ldrCfg.LoaderCgrCfg().FieldSeparator, strings.TrimPrefix(*dataPath, gprefix), *cfgPath)
-		if err != nil {
-			log.Fatal(err)
-		}
-	} else {
-		loader = engine.NewFileCSVStorage(ldrCfg.LoaderCgrCfg().FieldSeparator, *dataPath)
+	var loader engine.LoadReader
+	if loader, err = getLoader(ldrCfg); err != nil {
+		log.Fatal(err)
 	}
-
-	tpReader, err := engine.NewTpReader(dm.DataDB(), loader, ldrCfg.LoaderCgrCfg().TpID,
-		ldrCfg.GeneralCfg().DefaultTimezone, ldrCfg.LoaderCgrCfg().CachesConns, ldrCfg.LoaderCgrCfg().SchedulerConns)
-	if err != nil {
+	var tpReader *engine.TpReader
+	if tpReader, err = engine.NewTpReader(dataDB, loader,
+		ldrCfg.LoaderCgrCfg().TpID, ldrCfg.GeneralCfg().DefaultTimezone,
+		ldrCfg.LoaderCgrCfg().CachesConns,
+		ldrCfg.LoaderCgrCfg().SchedulerConns); err != nil {
 		log.Fatal(err)
 	}
 	if err = tpReader.LoadAll(); err != nil {
@@ -320,29 +337,25 @@ func main() {
 	}
 
 	if *remove {
-		if err := tpReader.RemoveFromDatabase(*verbose, *disableReverse); err != nil {
+		if err = tpReader.RemoveFromDatabase(*verbose, *disableReverse); err != nil {
 			log.Fatal("Could not delete from database: ", err)
 		}
 		return
 	}
 
 	// write maps to database
-	if err := tpReader.WriteToDatabase(*verbose, *disableReverse); err != nil {
+	if err = tpReader.WriteToDatabase(*verbose, *disableReverse); err != nil {
 		log.Fatal("Could not write to database: ", err)
 	}
-	caching := config.CgrConfig().GeneralCfg().DefaultCaching
-	if cachingArg != nil && *cachingArg != utils.EmptyString {
-		caching = *cachingArg
-	}
 	// reload cache
-	if err := tpReader.ReloadCache(caching, *verbose, &utils.ArgDispatcher{
+	if err = tpReader.ReloadCache(ldrCfg.GeneralCfg().DefaultCaching, *verbose, &utils.ArgDispatcher{
 		APIKey:  apiKey,
 		RouteID: routeID,
 	}); err != nil {
 		log.Fatal("Could not reload cache: ", err)
 	}
 	if len(ldrCfg.LoaderCgrCfg().SchedulerConns) != 0 {
-		if err := tpReader.ReloadScheduler(*verbose); err != nil {
+		if err = tpReader.ReloadScheduler(*verbose); err != nil {
 			log.Fatal("Could not reload scheduler: ", err)
 		}
 	}

config/config.go

@@ -1237,7 +1237,7 @@ func (*CGRConfig) loadConfigFromReader(rdr io.Reader, loadFuncs []func(jsnCfg *C
 
 // Reads all .json files out of a folder/subfolders and loads them up in lexical order
 func (cfg *CGRConfig) loadConfigFromPath(path string, loadFuncs []func(jsnCfg *CgrJsonCfg) error) (err error) {
-	if isUrl(path) {
+	if utils.IsURL(path) {
 		return cfg.loadConfigFromHTTP(path, loadFuncs) // prefix protocol
 	}
 	var fi os.FileInfo

config/libconfig.go

@@ -1,26 +0,0 @@
-/*
-Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
-Copyright (C) ITsysCOM GmbH
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package config
-
-import "strings"
-
-func isUrl(path string) bool {
-	return strings.HasPrefix(path, "https://") ||
-		strings.HasPrefix(path, "http://")
-}

config/libconfig_test.go

@@ -1,33 +0,0 @@
-/*
-Real-time Online/Offline Charging System (OCS) for Telecom & ISP environments
-Copyright (C) ITsysCOM GmbH
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-package config
-
-import "testing"
-
-func TestIsURL(t *testing.T) {
-	urls := map[string]bool{
-		"/etc/usr/":                           false,
-		"https://github.com/cgrates/cgrates/": true,
-		"http://github.com/cgrates/cgrates/i": true,
-	}
-	for url, expected := range urls {
-		if rply := isUrl(url); rply != expected {
-			t.Errorf("For: %q ,expected %v received: %v", url, expected, rply)
-		}
-	}
-}

engine/storage_csv.go

@@ -26,11 +26,13 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
 
+	"github.com/cgrates/cgrates/config"
 	"github.com/cgrates/cgrates/utils"
 	"golang.org/x/net/context"
 	"golang.org/x/oauth2"
@@ -38,6 +40,7 @@ import (
 	"google.golang.org/api/sheets/v4"
 )
 
+// CSVStorage the basic csv storage
 type CSVStorage struct {
 	sep       rune
 	generator func() csvReaderCloser
@@ -64,6 +67,7 @@ type CSVStorage struct {
 	dispatcherHostsFn        []string
 }
 
+// NewCSVStorage creates a CSV storege that takes the data from the paths specified
 func NewCSVStorage(sep rune,
 	destinationsFn, timingsFn, ratesFn, destinationratesFn,
 	destinationratetimingsFn, ratingprofilesFn, sharedgroupsFn,
@@ -97,55 +101,57 @@ func NewCSVStorage(sep rune,
 	}
 }
 
+// NewFileCSVStorage returns a csv storage that uses all files from the folder
 func NewFileCSVStorage(sep rune, dataPath string) *CSVStorage {
 	allFoldersPath, err := getAllFolders(dataPath)
 	if err != nil {
 		log.Fatal(err)
 	}
-	destinations_paths := appendName(allFoldersPath, utils.DestinationsCsv)
-	timings_paths := appendName(allFoldersPath, utils.TimingsCsv)
-	rates_paths := appendName(allFoldersPath, utils.RatesCsv)
-	destination_rates_paths := appendName(allFoldersPath, utils.DestinationRatesCsv)
-	rating_plans_paths := appendName(allFoldersPath, utils.RatingPlansCsv)
-	rating_profiles_paths := appendName(allFoldersPath, utils.RatingProfilesCsv)
-	shared_groups_paths := appendName(allFoldersPath, utils.SharedGroupsCsv)
-	actions_paths := appendName(allFoldersPath, utils.ActionsCsv)
-	action_plans_paths := appendName(allFoldersPath, utils.ActionPlansCsv)
-	action_triggers_paths := appendName(allFoldersPath, utils.ActionTriggersCsv)
-	account_actions_paths := appendName(allFoldersPath, utils.AccountActionsCsv)
-	resources_paths := appendName(allFoldersPath, utils.ResourcesCsv)
-	stats_paths := appendName(allFoldersPath, utils.StatsCsv)
-	thresholds_paths := appendName(allFoldersPath, utils.ThresholdsCsv)
-	filters_paths := appendName(allFoldersPath, utils.FiltersCsv)
-	suppliers_paths := appendName(allFoldersPath, utils.SuppliersCsv)
-	attributes_paths := appendName(allFoldersPath, utils.AttributesCsv)
-	chargers_paths := appendName(allFoldersPath, utils.ChargersCsv)
-	dispatcherprofiles_paths := appendName(allFoldersPath, utils.DispatcherProfilesCsv)
-	dispatcherhosts_paths := appendName(allFoldersPath, utils.DispatcherHostsCsv)
+	destinationsPaths := appendName(allFoldersPath, utils.DestinationsCsv)
+	timingsPaths := appendName(allFoldersPath, utils.TimingsCsv)
+	ratesPaths := appendName(allFoldersPath, utils.RatesCsv)
+	destinationRatesPaths := appendName(allFoldersPath, utils.DestinationRatesCsv)
+	ratingPlansPaths := appendName(allFoldersPath, utils.RatingPlansCsv)
+	ratingProfilesPaths := appendName(allFoldersPath, utils.RatingProfilesCsv)
+	sharedGroupsPaths := appendName(allFoldersPath, utils.SharedGroupsCsv)
+	actionsPaths := appendName(allFoldersPath, utils.ActionsCsv)
+	actionPlansPaths := appendName(allFoldersPath, utils.ActionPlansCsv)
+	actionTriggersPaths := appendName(allFoldersPath, utils.ActionTriggersCsv)
+	accountActionsPaths := appendName(allFoldersPath, utils.AccountActionsCsv)
+	resourcesPaths := appendName(allFoldersPath, utils.ResourcesCsv)
+	statsPaths := appendName(allFoldersPath, utils.StatsCsv)
+	thresholdsPaths := appendName(allFoldersPath, utils.ThresholdsCsv)
+	filtersPaths := appendName(allFoldersPath, utils.FiltersCsv)
+	suppliersPaths := appendName(allFoldersPath, utils.SuppliersCsv)
+	attributesPaths := appendName(allFoldersPath, utils.AttributesCsv)
+	chargersPaths := appendName(allFoldersPath, utils.ChargersCsv)
+	dispatcherprofilesPaths := appendName(allFoldersPath, utils.DispatcherProfilesCsv)
+	dispatcherhostsPaths := appendName(allFoldersPath, utils.DispatcherHostsCsv)
 	return NewCSVStorage(sep,
-		destinations_paths,
-		timings_paths,
-		rates_paths,
-		destination_rates_paths,
-		rating_plans_paths,
-		rating_profiles_paths,
-		shared_groups_paths,
-		actions_paths,
-		action_plans_paths,
-		action_triggers_paths,
-		account_actions_paths,
-		resources_paths,
-		stats_paths,
-		thresholds_paths,
-		filters_paths,
-		suppliers_paths,
-		attributes_paths,
-		chargers_paths,
-		dispatcherprofiles_paths,
-		dispatcherhosts_paths,
+		destinationsPaths,
+		timingsPaths,
+		ratesPaths,
+		destinationRatesPaths,
+		ratingPlansPaths,
+		ratingProfilesPaths,
+		sharedGroupsPaths,
+		actionsPaths,
+		actionPlansPaths,
+		actionTriggersPaths,
+		accountActionsPaths,
+		resourcesPaths,
+		statsPaths,
+		thresholdsPaths,
+		filtersPaths,
+		suppliersPaths,
+		attributesPaths,
+		chargersPaths,
+		dispatcherprofilesPaths,
+		dispatcherhostsPaths,
 	)
 }
 
+// NewStringCSVStorage creates a csv storage from strings
 func NewStringCSVStorage(sep rune,
 	destinationsFn, timingsFn, ratesFn, destinationratesFn,
 	destinationratetimingsFn, ratingprofilesFn, sharedgroupsFn,
@@ -165,12 +171,13 @@ func NewStringCSVStorage(sep rune,
 	return c
 }
 
-func NewGoogleCSVStorage(sep rune, spreadsheetId, cfgPath string) (*CSVStorage, error) {
+// NewGoogleCSVStorage creates a csv storege from google sheets
+func NewGoogleCSVStorage(sep rune, spreadsheetID, cfgPath string) (*CSVStorage, error) {
 	sht, err := newSheet(cfgPath)
 	if err != nil {
 		return nil, err
 	}
-	sheetNames, err := getSpreatsheetTabs(spreadsheetId, sht)
+	sheetNames, err := getSpreatsheetTabs(spreadsheetID, sht)
 	if err != nil {
 		return nil, err
 	}
@@ -203,13 +210,141 @@ func NewGoogleCSVStorage(sep rune, spreadsheetId, cfgPath string) (*CSVStorage,
 		getIfExist(utils.DispatcherHosts))
 	c.generator = func() csvReaderCloser {
 		return &csvGoogle{
-			spreadsheetId: spreadsheetId,
+			spreadsheetID: spreadsheetID,
 			srv:           sht,
 		}
 	}
 	return c, nil
 }
 
+// NewURLCSVStorage returns a CSVStorage that can parse URLs
+func NewURLCSVStorage(sep rune, dataPath string) *CSVStorage {
+	var destinationsPaths []string
+	var timingsPaths []string
+	var ratesPaths []string
+	var destinationRatesPaths []string
+	var ratingPlansPaths []string
+	var ratingProfilesPaths []string
+	var sharedGroupsPaths []string
+	var actionsPaths []string
+	var actionPlansPaths []string
+	var actionTriggersPaths []string
+	var accountActionsPaths []string
+	var resourcesPaths []string
+	var statsPaths []string
+	var thresholdsPaths []string
+	var filtersPaths []string
+	var suppliersPaths []string
+	var attributesPaths []string
+	var chargersPaths []string
+	var dispatcherprofilesPaths []string
+	var dispatcherhostsPaths []string
+
+	for _, baseURL := range strings.Split(dataPath, utils.INFIELD_SEP) {
+		if !strings.HasSuffix(baseURL, utils.CSVSuffix) {
+			destinationsPaths = append(destinationsPaths, joinURL(baseURL, utils.DestinationsCsv))
+			timingsPaths = append(timingsPaths, joinURL(baseURL, utils.TimingsCsv))
+			ratesPaths = append(ratesPaths, joinURL(baseURL, utils.RatesCsv))
+			destinationRatesPaths = append(destinationRatesPaths, joinURL(baseURL, utils.DestinationRatesCsv))
+			ratingPlansPaths = append(ratingPlansPaths, joinURL(baseURL, utils.RatingPlansCsv))
+			ratingProfilesPaths = append(ratingProfilesPaths, joinURL(baseURL, utils.RatingProfilesCsv))
+			sharedGroupsPaths = append(sharedGroupsPaths, joinURL(baseURL, utils.SharedGroupsCsv))
+			actionsPaths = append(actionsPaths, joinURL(baseURL, utils.ActionsCsv))
+			actionPlansPaths = append(actionPlansPaths, joinURL(baseURL, utils.ActionPlansCsv))
+			actionTriggersPaths = append(actionTriggersPaths, joinURL(baseURL, utils.ActionTriggersCsv))
+			accountActionsPaths = append(accountActionsPaths, joinURL(baseURL, utils.AccountActionsCsv))
+			resourcesPaths = append(resourcesPaths, joinURL(baseURL, utils.ResourcesCsv))
+			statsPaths = append(statsPaths, joinURL(baseURL, utils.StatsCsv))
+			thresholdsPaths = append(thresholdsPaths, joinURL(baseURL, utils.ThresholdsCsv))
+			filtersPaths = append(filtersPaths, joinURL(baseURL, utils.FiltersCsv))
+			suppliersPaths = append(suppliersPaths, joinURL(baseURL, utils.SuppliersCsv))
+			attributesPaths = append(attributesPaths, joinURL(baseURL, utils.AttributesCsv))
+			chargersPaths = append(chargersPaths, joinURL(baseURL, utils.ChargersCsv))
+			dispatcherprofilesPaths = append(dispatcherprofilesPaths, joinURL(baseURL, utils.DispatcherProfilesCsv))
+			dispatcherhostsPaths = append(dispatcherhostsPaths, joinURL(baseURL, utils.DispatcherHostsCsv))
+			continue
+		}
+		switch {
+		case strings.HasSuffix(baseURL, utils.DestinationsCsv):
+			destinationsPaths = append(destinationsPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.TimingsCsv):
+			timingsPaths = append(timingsPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.RatesCsv):
+			ratesPaths = append(ratesPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.DestinationRatesCsv):
+			destinationRatesPaths = append(destinationRatesPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.RatingPlansCsv):
+			ratingPlansPaths = append(ratingPlansPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.RatingProfilesCsv):
+			ratingProfilesPaths = append(ratingProfilesPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.SharedGroupsCsv):
+			sharedGroupsPaths = append(sharedGroupsPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.ActionsCsv):
+			actionsPaths = append(actionsPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.ActionPlansCsv):
+			actionPlansPaths = append(actionPlansPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.ActionTriggersCsv):
+			actionTriggersPaths = append(actionTriggersPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.AccountActionsCsv):
+			accountActionsPaths = append(accountActionsPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.ResourcesCsv):
+			resourcesPaths = append(resourcesPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.StatsCsv):
+			statsPaths = append(statsPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.ThresholdsCsv):
+			thresholdsPaths = append(thresholdsPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.FiltersCsv):
+			filtersPaths = append(filtersPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.SuppliersCsv):
+			suppliersPaths = append(suppliersPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.AttributesCsv):
+			attributesPaths = append(attributesPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.ChargersCsv):
+			chargersPaths = append(chargersPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.DispatcherProfilesCsv):
+			dispatcherprofilesPaths = append(dispatcherprofilesPaths, baseURL)
+		case strings.HasSuffix(baseURL, utils.DispatcherHostsCsv):
+			dispatcherhostsPaths = append(dispatcherhostsPaths, baseURL)
+		}
+	}
+
+	c := NewCSVStorage(sep,
+		destinationsPaths,
+		timingsPaths,
+		ratesPaths,
+		destinationRatesPaths,
+		ratingPlansPaths,
+		ratingProfilesPaths,
+		sharedGroupsPaths,
+		actionsPaths,
+		actionPlansPaths,
+		actionTriggersPaths,
+		accountActionsPaths,
+		resourcesPaths,
+		statsPaths,
+		thresholdsPaths,
+		filtersPaths,
+		suppliersPaths,
+		attributesPaths,
+		chargersPaths,
+		dispatcherprofilesPaths,
+		dispatcherhostsPaths,
+	)
+	c.generator = func() csvReaderCloser {
+		return &csvURL{}
+	}
+	return c
+}
+
+func joinURL(baseURL, fn string) string {
+	u, err := url.Parse(baseURL)
+	if err != nil {
+		return baseURL + fn
+	}
+	u.Path = path.Join(u.Path, fn)
+	return u.String()
+}
+
 func getAllFolders(inPath string) (paths []string, err error) {
 	err = filepath.Walk(inPath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -225,8 +360,8 @@ func getAllFolders(inPath string) (paths []string, err error) {
 
 func appendName(paths []string, fileName string) (out []string) {
 	out = make([]string, len(paths))
-	for i, path_ := range paths {
-		out[i] = path.Join(path_, fileName)
+	for i, basePath := range paths {
+		out[i] = path.Join(basePath, fileName)
 	}
 	return
 }
@@ -665,9 +800,9 @@ func newSheet(configPath string) (sht *sheets.Service, err error) { //*google_ap
 	return
 }
 
-func getSpreatsheetTabs(spreadsheetId string, srv *sheets.Service) (sheetsName map[string]struct{}, err error) {
+func getSpreatsheetTabs(spreadsheetID string, srv *sheets.Service) (sheetsName map[string]struct{}, err error) {
 	sheetsName = make(map[string]struct{})
-	sht, err := srv.Spreadsheets.Get(spreadsheetId).Do()
+	sht, err := srv.Spreadsheets.Get(spreadsheetID).Do()
 	if err != nil {
 		err = fmt.Errorf("Unable get the information about spreadsheet because: %v", err)
 		return
@@ -679,7 +814,7 @@ func getSpreatsheetTabs(spreadsheetId string, srv *sheets.Service) (sheetsName m
 }
 
 type csvGoogle struct {
-	spreadsheetId string
+	spreadsheetID string
 	srv           *sheets.Service
 	response      *sheets.ValueRange
 	indx          int
@@ -687,7 +822,7 @@ type csvGoogle struct {
 }
 
 func (c *csvGoogle) Open(data string, sep rune, nrFields int) (err error) {
-	c.response, err = c.srv.Spreadsheets.Values.Get(c.spreadsheetId, data).Do()
+	c.response, err = c.srv.Spreadsheets.Values.Get(c.spreadsheetID, data).Do()
 	if err != nil {
 		return
 	}
@@ -732,3 +867,43 @@ func (c *csvGoogle) Read() (record []string, err error) {
 
 func (c *csvGoogle) Close() { // no need for close
 }
+
+type csvURL struct {
+	csvReader *csv.Reader
+	page      io.ReadCloser
+}
+
+func (c *csvURL) Open(fn string, sep rune, nrFields int) (err error) {
+	if _, err = url.ParseRequestURI(fn); err != nil {
+		return
+	}
+	var myClient = &http.Client{
+		Timeout: config.CgrConfig().GeneralCfg().ReplyTimeout,
+	}
+	var req *http.Response
+	req, err = myClient.Get(fn)
+	if err != nil {
+		return utils.ErrPathNotReachable(fn)
+	}
+	if req.StatusCode != http.StatusOK {
+		return utils.ErrNotFound
+	}
+	c.page = req.Body
+
+	c.csvReader = csv.NewReader(c.page)
+	c.csvReader.Comma = sep
+	c.csvReader.Comment = utils.COMMENT_CHAR
+	c.csvReader.FieldsPerRecord = nrFields
+	c.csvReader.TrailingComma = true
+	return
+}
+
+func (c *csvURL) Read() (record []string, err error) {
+	return c.csvReader.Read()
+}
+
+func (c *csvURL) Close() {
+	if c.page != nil {
+		c.page.Close()
+	}
+}

utils/coreutils.go

@@ -942,3 +942,9 @@ type LoadIDsWithArgDispatcher struct {
 	TenantArg
 	*ArgDispatcher
 }
+
+// IsURL returns if the path is an URL
+func IsURL(path string) bool {
+	return strings.HasPrefix(path, "https://") ||
+		strings.HasPrefix(path, "http://")
+}

utils/coreutils_test.go

@@ -1349,3 +1349,16 @@ func TestGetPathIndex(t *testing.T) {
 		t.Errorf("Expecting: nil, received: %+v", *index)
 	}
 }
+
+func TestIsURL(t *testing.T) {
+	urls := map[string]bool{
+		"/etc/usr/":                           false,
+		"https://github.com/cgrates/cgrates/": true,
+		"http://github.com/cgrates/cgrates/i": true,
+	}
+	for url, expected := range urls {
+		if rply := IsURL(url); rply != expected {
+			t.Errorf("For: %q ,expected %v received: %v", url, expected, rply)
+		}
+	}
+}