Adding Account and User attributes to tutorial/Users.csv, removing .Debug through code

Fix account command in the docs

.gitignore

@@ -12,3 +12,4 @@ bin
 dean*
 data/vagrant/.vagrant
 data/vagrant/vagrant_ansible_inventory_default
+data/tutorials/fs_evsock/freeswitch/etc/freeswitch/

.travis.yml

@@ -1,9 +1,7 @@
 language: go
 
 go:
-  - 1.2
-  - 1.3
-  - tip
+  - 1.4
 
 script: $TRAVIS_BUILD_DIR/test.sh
 

CONTRIBUTING.md

@@ -0,0 +1,26 @@
+## Contributor Agreement (required for pull requests)
+
+While the CGRateS project is open source under the GPLv3 license, contributions
+back to this project which contain code must be made available to ITsysCOM GmbH under the terms of the MIT license.
+
+### What does this mean?
+
+This means ITsysCOM GmbH can do the following with code contributed
+back to the project from the open source community.
+
+1. Include the code in future CGRateS releases.
+2. Include the code, in original or modified form, in other products created by ITsysCOM GmbH, **including closed-source products**.
+
+### How do I agree to these terms?
+
+To agree to these terms, sign the contributing agreement in `CONTRIBUTORS.md` and
+include the change as a commit with your pull request. **Pull requests which contain
+commits from users not explicitly listed in the `CONTRIBUTORS.md` file will be
+rejected *without review*.**
+
+* You only need to sign the `CONTRIBUTORS.md` file prior to the first time you
+  submit a pull request (or include it as part of the pull request itself).
+* The submitter of a pull request which adds a user to the `CONTRIBUTORS.md` file
+  *must* match the user information which was added to the `CONTRIBUTORS.md` file.
+  For example, only @danbogos can send a pull request containing a commit which adds
+  @danbogos to the `CONTRIBUTORS.md` file.

CONTRIBUTORS.md

@@ -0,0 +1,42 @@
+## Contributors
+
+By signing this agreement, the undersigned agree that contributions made to the CGRateS
+project are provided under the terms of the MIT license, included below. For additional
+information, please see the [`CONTRIBUTING.md`](CONTRIBUTING.md) file.
+
+**Pull requests which add a user to this file may only be submitted by the user who was added to the file.**
+
+### Contributor License Text
+
+> The MIT License (MIT)
+>
+> Copyright © <year> <copyright holders>
+>
+> Permission is hereby granted, free of charge, to any person obtaining a copy
+> of this software and associated documentation files (the "Software"), to deal
+> in the Software without restriction, including without limitation the rights
+> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+> copies of the Software, and to permit persons to whom the Software is
+> furnished to do so, subject to the following conditions:
+>
+> The above copyright notice and this permission notice shall be included in
+> all copies or substantial portions of the Software.
+>
+> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+> THE SOFTWARE.
+
+### Undersigned
+
+| GitHub Username | Full Name |
+| --------------- | --------- |
+| @danbogos | Dan Christian Bogos |
+| @eloycoto | Eloy Coto Pereiro |
+
+<!-- to sign, include a single line above this comment containing the following text:
+| @username | First Last |
+-->

README.md

@@ -26,4 +26,3 @@ API reference [godoc](http://godoc.org/github.com/cgrates/cgrates/apier)
 
 Also check [irc.freenode.net #cgrates](irc://irc.freenode.net:6667/cgrates)  ([Webchat](http://webchat.freenode.net?randomnick=1&channels=%23cgrates)) and [Google group](https://groups.google.com/forum/#!forum/cgrates) for a more real-time support.
 
-[![Analytics](https://ga-beacon.appspot.com/UA-10073547-11/cgrates/readme)](https://github.com/igrigorik/ga-beacon)

apier/apier.go

@@ -1,798 +0,0 @@
-/*
-Real-time Charging System for Telecom & ISP environments
-Copyright (C) ITsysCOM GmbH
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package apier
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"path"
-
-	"github.com/cgrates/cgrates/cache2go"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/scheduler"
-	"github.com/cgrates/cgrates/utils"
-)
-
-const (
-	OK = "OK"
-)
-
-type ApierV1 struct {
-	StorDb      engine.LoadStorage
-	RatingDb    engine.RatingStorage
-	AccountDb   engine.AccountingStorage
-	CdrDb       engine.CdrStorage
-	LogDb       engine.LogStorage
-	Sched       *scheduler.Scheduler
-	Config      *config.CGRConfig
-	Responder   *engine.Responder
-	CdrStatsSrv *engine.Stats
-}
-
-func (self *ApierV1) GetDestination(dstId string, reply *engine.Destination) error {
-	if dst, err := self.RatingDb.GetDestination(dstId); err != nil {
-		return errors.New(utils.ERR_NOT_FOUND)
-	} else {
-		*reply = *dst
-	}
-	return nil
-}
-
-type AttrSetDestination struct { //ToDo
-	Id        string
-	Prefixes  []string
-	Overwrite bool
-}
-
-func (self *ApierV1) GetRatingPlan(rplnId string, reply *engine.RatingPlan) error {
-	if rpln, err := self.RatingDb.GetRatingPlan(rplnId, false); err != nil {
-		return errors.New(utils.ERR_NOT_FOUND)
-	} else {
-		*reply = *rpln
-	}
-	return nil
-}
-
-// Get balance
-func (self *ApierV1) GetAccount(attr *utils.AttrGetAccount, reply *engine.Account) error {
-	tag := fmt.Sprintf("%s:%s:%s", attr.Direction, attr.Tenant, attr.Account)
-	userBalance, err := self.AccountDb.GetAccount(tag)
-	if err != nil {
-		return err
-	}
-
-	*reply = *userBalance
-	return nil
-}
-
-type AttrAddBalance struct {
-	Tenant        string
-	Account       string
-	BalanceType   string
-	Direction     string
-	Value         float64
-	ExpiryTime    string
-	RatingSubject string
-	DestinationId string
-	Weight        float64
-	SharedGroup   string
-	Overwrite     bool // When true it will reset if the balance is already there
-}
-
-func (self *ApierV1) AddBalance(attr *AttrAddBalance, reply *string) error {
-	expTime, err := utils.ParseDate(attr.ExpiryTime)
-	if err != nil {
-		*reply = err.Error()
-		return err
-	}
-	tag := fmt.Sprintf("%s:%s:%s", attr.Direction, attr.Tenant, attr.Account)
-	if _, err := self.AccountDb.GetAccount(tag); err != nil {
-		// create user balance if not exists
-		ub := &engine.Account{
-			Id: tag,
-		}
-		if err := self.AccountDb.SetAccount(ub); err != nil {
-			*reply = err.Error()
-			return err
-		}
-	}
-	at := &engine.ActionTiming{
-		AccountIds: []string{tag},
-	}
-
-	if attr.Direction == "" {
-		attr.Direction = engine.OUTBOUND
-	}
-	aType := engine.DEBIT
-	// reverse the sign as it is a debit
-	attr.Value = -attr.Value
-
-	if attr.Overwrite {
-		aType = engine.DEBIT_RESET
-	}
-	at.SetActions(engine.Actions{
-		&engine.Action{
-			ActionType:  aType,
-			BalanceType: attr.BalanceType,
-			Direction:   attr.Direction,
-			Balance: &engine.Balance{
-				Value:          attr.Value,
-				ExpirationDate: expTime,
-				RatingSubject:  attr.RatingSubject,
-				DestinationId:  attr.DestinationId,
-				Weight:         attr.Weight,
-				SharedGroup:    attr.SharedGroup,
-			},
-		},
-	})
-	if err := at.Execute(); err != nil {
-		*reply = err.Error()
-		return err
-	}
-	*reply = OK
-	return nil
-}
-
-type AttrExecuteAction struct {
-	Direction string
-	Tenant    string
-	Account   string
-	ActionsId string
-}
-
-func (self *ApierV1) ExecuteAction(attr *AttrExecuteAction, reply *string) error {
-	tag := fmt.Sprintf("%s:%s:%s", attr.Direction, attr.Tenant, attr.Account)
-	at := &engine.ActionTiming{
-		AccountIds: []string{tag},
-		ActionsId:  attr.ActionsId,
-	}
-
-	if err := at.Execute(); err != nil {
-		*reply = err.Error()
-		return err
-	}
-	*reply = OK
-	return nil
-}
-
-type AttrLoadRatingPlan struct {
-	TPid         string
-	RatingPlanId string
-}
-
-// Process dependencies and load a specific rating plan from storDb into dataDb.
-func (self *ApierV1) LoadRatingPlan(attrs AttrLoadRatingPlan, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingPlanId"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	dbReader := engine.NewDbReader(self.StorDb, self.RatingDb, self.AccountDb, attrs.TPid)
-	if loaded, err := dbReader.LoadRatingPlanByTag(attrs.RatingPlanId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if !loaded {
-		return errors.New("NOT_FOUND")
-	}
-	//Automatic cache of the newly inserted rating plan
-	didNotChange := []string{}
-	if err := self.RatingDb.CacheRating(nil, nil, didNotChange, didNotChange, didNotChange); err != nil {
-		return err
-	}
-	*reply = OK
-	return nil
-}
-
-// Process dependencies and load a specific rating profile from storDb into dataDb.
-func (self *ApierV1) LoadRatingProfile(attrs utils.TPRatingProfile, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "TOR", "Direction", "Subject"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	dbReader := engine.NewDbReader(self.StorDb, self.RatingDb, self.AccountDb, attrs.TPid)
-	if err := dbReader.LoadRatingProfileFiltered(&attrs); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	//Automatic cache of the newly inserted rating profile
-	didNotChange := []string{}
-	if err := self.RatingDb.CacheRating(didNotChange, didNotChange, []string{engine.RATING_PROFILE_PREFIX + attrs.KeyId()}, didNotChange, didNotChange); err != nil {
-		return err
-	}
-	*reply = OK
-	return nil
-}
-
-type AttrSetRatingProfile struct {
-	Tenant                string                      // Tenant's Id
-	Category              string                      // TypeOfRecord
-	Direction             string                      // Traffic direction, OUT is the only one supported for now
-	Subject               string                      // Rating subject, usually the same as account
-	Overwrite             bool                        // Overwrite if exists
-	RatingPlanActivations []*utils.TPRatingActivation // Activate rating plans at specific time
-}
-
-// Sets a specific rating profile working with data directly in the RatingDb without involving storDb
-func (self *ApierV1) SetRatingProfile(attrs AttrSetRatingProfile, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "TOR", "Direction", "Subject", "RatingPlanActivations"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	for _, rpa := range attrs.RatingPlanActivations {
-		if missing := utils.MissingStructFields(rpa, []string{"ActivationTime", "RatingPlanId"}); len(missing) != 0 {
-			return fmt.Errorf("%s:RatingPlanActivation:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-		}
-	}
-	tpRpf := utils.TPRatingProfile{Tenant: attrs.Tenant, Category: attrs.Category, Direction: attrs.Direction, Subject: attrs.Subject}
-	keyId := tpRpf.KeyId()
-	if !attrs.Overwrite {
-		if exists, err := self.RatingDb.HasData(engine.RATING_PROFILE_PREFIX, keyId); err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-		} else if exists {
-			return errors.New(utils.ERR_EXISTS)
-		}
-	}
-	rpfl := &engine.RatingProfile{Id: keyId, RatingPlanActivations: make(engine.RatingPlanActivations, len(attrs.RatingPlanActivations))}
-	for idx, ra := range attrs.RatingPlanActivations {
-		at, err := utils.ParseDate(ra.ActivationTime)
-		if err != nil {
-			return fmt.Errorf(fmt.Sprintf("%s:Cannot parse activation time from %v", utils.ERR_SERVER_ERROR, ra.ActivationTime))
-		}
-		if exists, err := self.RatingDb.HasData(engine.RATING_PLAN_PREFIX, ra.RatingPlanId); err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-		} else if !exists {
-			return fmt.Errorf(fmt.Sprintf("%s:RatingPlanId:%s", utils.ERR_NOT_FOUND, ra.RatingPlanId))
-		}
-		rpfl.RatingPlanActivations[idx] = &engine.RatingPlanActivation{ActivationTime: at, RatingPlanId: ra.RatingPlanId,
-			FallbackKeys: utils.FallbackSubjKeys(tpRpf.Direction, tpRpf.Tenant, tpRpf.Category, ra.FallbackSubjects)}
-	}
-	if err := self.RatingDb.SetRatingProfile(rpfl); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	//Automatic cache of the newly inserted rating profile
-	didNotChange := []string{}
-	if err := self.RatingDb.CacheRating(didNotChange, didNotChange, []string{engine.RATING_PROFILE_PREFIX + keyId}, didNotChange, didNotChange); err != nil {
-		return err
-	}
-	*reply = OK
-	return nil
-}
-
-type AttrSetActions struct {
-	ActionsId string            // Actions id
-	Overwrite bool              // If previously defined, will be overwritten
-	Actions   []*utils.TPAction // Set of actions this Actions profile will perform
-}
-
-func (self *ApierV1) SetActions(attrs AttrSetActions, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"ActionsId", "Actions"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	for _, action := range attrs.Actions {
-		requiredFields := []string{"Identifier", "Weight"}
-		if action.BalanceType != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions
-			requiredFields = append(requiredFields, "Direction", "Units")
-		}
-		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
-			return fmt.Errorf("%s:Action:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
-		}
-	}
-	if !attrs.Overwrite {
-		if exists, err := self.AccountDb.HasData(engine.ACTION_PREFIX, attrs.ActionsId); err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-		} else if exists {
-			return errors.New(utils.ERR_EXISTS)
-		}
-	}
-	storeActions := make(engine.Actions, len(attrs.Actions))
-	for idx, apiAct := range attrs.Actions {
-		a := &engine.Action{
-			Id:               utils.GenUUID(),
-			ActionType:       apiAct.Identifier,
-			BalanceType:      apiAct.BalanceType,
-			Direction:        apiAct.Direction,
-			Weight:           apiAct.Weight,
-			ExpirationString: apiAct.ExpiryTime,
-			ExtraParameters:  apiAct.ExtraParameters,
-			Balance: &engine.Balance{
-				Uuid:          utils.GenUUID(),
-				Value:         apiAct.Units,
-				Weight:        apiAct.BalanceWeight,
-				DestinationId: apiAct.DestinationId,
-				RatingSubject: apiAct.RatingSubject,
-				SharedGroup:   apiAct.SharedGroup,
-			},
-		}
-		storeActions[idx] = a
-	}
-	if err := self.AccountDb.SetActions(attrs.ActionsId, storeActions); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	didNotChange := []string{}
-	self.AccountDb.CacheAccounting(nil, didNotChange, didNotChange, didNotChange)
-	*reply = OK
-	return nil
-}
-
-// Retrieves actions attached to specific ActionsId within cache
-func (self *ApierV1) GetActions(actsId string, reply *[]*utils.TPAction) error {
-	if len(actsId) == 0 {
-		return fmt.Errorf("%s:ActionsId", utils.ERR_MANDATORY_IE_MISSING, actsId)
-	}
-	acts := make([]*utils.TPAction, 0)
-	engActs, err := self.AccountDb.GetActions(actsId, false)
-	if err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	for _, engAct := range engActs {
-		act := &utils.TPAction{Identifier: engAct.ActionType,
-			BalanceType:     engAct.BalanceType,
-			Direction:       engAct.Direction,
-			ExpiryTime:      engAct.ExpirationString,
-			ExtraParameters: engAct.ExtraParameters,
-			Weight:          engAct.Weight,
-		}
-		if engAct.Balance != nil {
-			act.Units = engAct.Balance.Value
-			act.DestinationId = engAct.Balance.DestinationId
-			act.RatingSubject = engAct.Balance.RatingSubject
-			act.SharedGroup = engAct.Balance.SharedGroup
-			act.BalanceWeight = engAct.Balance.Weight
-		}
-		acts = append(acts, act)
-	}
-	*reply = acts
-	return nil
-}
-
-type AttrSetActionPlan struct {
-	Id              string             // Profile id
-	ActionPlan      []*ApiActionTiming // Set of actions this Actions profile will perform
-	Overwrite       bool               // If previously defined, will be overwritten
-	ReloadScheduler bool               // Enables automatic reload of the scheduler (eg: useful when adding a single action timing)
-}
-
-type ApiActionTiming struct {
-	ActionsId string  // Actions id
-	Years     string  // semicolon separated list of years this timing is valid on, *any or empty supported
-	Months    string  // semicolon separated list of months this timing is valid on, *any or empty supported
-	MonthDays string  // semicolon separated list of month's days this timing is valid on, *any or empty supported
-	WeekDays  string  // semicolon separated list of week day names this timing is valid on *any or empty supported
-	Time      string  // String representing the time this timing starts on, *asap supported
-	Weight    float64 // Binding's weight
-}
-
-func (self *ApierV1) SetActionPlan(attrs AttrSetActionPlan, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"Id", "ActionPlan"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	for _, at := range attrs.ActionPlan {
-		requiredFields := []string{"ActionsId", "Time", "Weight"}
-		if missing := utils.MissingStructFields(at, requiredFields); len(missing) != 0 {
-			return fmt.Errorf("%s:Action:%s:%v", utils.ERR_MANDATORY_IE_MISSING, at.ActionsId, missing)
-		}
-	}
-	if !attrs.Overwrite {
-		if exists, err := self.AccountDb.HasData(engine.ACTION_TIMING_PREFIX, attrs.Id); err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-		} else if exists {
-			return errors.New(utils.ERR_EXISTS)
-		}
-	}
-	storeAtms := make(engine.ActionPlan, len(attrs.ActionPlan))
-	for idx, apiAtm := range attrs.ActionPlan {
-		if exists, err := self.AccountDb.HasData(engine.ACTION_PREFIX, apiAtm.ActionsId); err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-		} else if !exists {
-			return fmt.Errorf("%s:%s", utils.ERR_BROKEN_REFERENCE, err.Error())
-		}
-		timing := new(engine.RITiming)
-		timing.Years.Parse(apiAtm.Years, ";")
-		timing.Months.Parse(apiAtm.Months, ";")
-		timing.MonthDays.Parse(apiAtm.MonthDays, ";")
-		timing.WeekDays.Parse(apiAtm.WeekDays, ";")
-		timing.StartTime = apiAtm.Time
-		at := &engine.ActionTiming{
-			Uuid:      utils.GenUUID(),
-			Id:        attrs.Id,
-			Weight:    apiAtm.Weight,
-			Timing:    &engine.RateInterval{Timing: timing},
-			ActionsId: apiAtm.ActionsId,
-		}
-		storeAtms[idx] = at
-	}
-	if err := self.AccountDb.SetActionTimings(attrs.Id, storeAtms); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	if attrs.ReloadScheduler {
-		if self.Sched == nil {
-			return errors.New("SCHEDULER_NOT_ENABLED")
-		}
-		self.Sched.LoadActionTimings(self.AccountDb)
-		self.Sched.Restart()
-	}
-	*reply = OK
-	return nil
-}
-
-type AttrAddActionTrigger struct {
-	Tenant               string
-	Account              string
-	Direction            string
-	BalanceType          string
-	ThresholdType        string
-	ThresholdValue       float64
-	DestinationId        string
-	BalanceRatingSubject string //ToDo
-	BalanceWeight        float64
-	BalanceExpiryTime    string
-	BalanceSharedGroup   string //ToDo
-	Weight               float64
-	ActionsId            string
-}
-
-func (self *ApierV1) AddTriggeredAction(attr AttrAddActionTrigger, reply *string) error {
-	if attr.Direction == "" {
-		attr.Direction = engine.OUTBOUND
-	}
-	balExpiryTime, err := utils.ParseTimeDetectLayout(attr.BalanceExpiryTime)
-	if err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	at := &engine.ActionTrigger{
-		Id:                    utils.GenUUID(),
-		BalanceType:           attr.BalanceType,
-		Direction:             attr.Direction,
-		ThresholdType:         attr.ThresholdType,
-		ThresholdValue:        attr.ThresholdValue,
-		DestinationId:         attr.DestinationId,
-		BalanceWeight:         attr.BalanceWeight,
-		BalanceExpirationDate: balExpiryTime,
-		Weight:                attr.Weight,
-		ActionsId:             attr.ActionsId,
-		Executed:              false,
-	}
-
-	tag := utils.AccountKey(attr.Tenant, attr.Account, attr.Direction)
-	_, err = engine.AccLock.Guard(tag, func() (float64, error) {
-		userBalance, err := self.AccountDb.GetAccount(tag)
-		if err != nil {
-			return 0, err
-		}
-
-		userBalance.ActionTriggers = append(userBalance.ActionTriggers, at)
-
-		if err = self.AccountDb.SetAccount(userBalance); err != nil {
-			return 0, err
-		}
-		return 0, nil
-	})
-	if err != nil {
-		*reply = err.Error()
-		return err
-	}
-	*reply = OK
-	return nil
-}
-
-type AttrResetTriggeredAction struct {
-	Tenant               string
-	Account              string
-	Direction            string
-	BalanceType          string
-	ThresholdType        string
-	ThresholdValue       float64
-	DestinationId        string
-	BalanceWeight        float64
-	BalanceRatingSubject string
-	BalanceSharedGroup   string
-}
-
-func (self *ApierV1) ResetTriggeredActions(attr AttrResetTriggeredAction, reply *string) error {
-	if attr.Direction == "" {
-		attr.Direction = engine.OUTBOUND
-	}
-	extraParameters, err := json.Marshal(struct {
-		ThresholdType        string
-		ThresholdValue       float64
-		DestinationId        string
-		BalanceWeight        float64
-		BalanceRatingSubject string
-		BalanceSharedGroup   string
-	}{
-		attr.ThresholdType,
-		attr.ThresholdValue,
-		attr.DestinationId,
-		attr.BalanceWeight,
-		attr.BalanceRatingSubject,
-		attr.BalanceSharedGroup,
-	})
-	if err != nil {
-		*reply = err.Error()
-		return err
-	}
-	a := &engine.Action{
-		BalanceType:     attr.BalanceType,
-		Direction:       attr.Direction,
-		ExtraParameters: string(extraParameters),
-	}
-	accID := utils.AccountKey(attr.Tenant, attr.Account, attr.Direction)
-	_, err = engine.AccLock.Guard(accID, func() (float64, error) {
-		acc, err := self.AccountDb.GetAccount(accID)
-		if err != nil {
-			return 0, err
-		}
-
-		acc.ResetActionTriggers(a)
-
-		if err = self.AccountDb.SetAccount(acc); err != nil {
-			return 0, err
-		}
-		return 0, nil
-	})
-	if err != nil {
-		*reply = err.Error()
-		return err
-	}
-	*reply = OK
-	return nil
-}
-
-// Process dependencies and load a specific AccountActions profile from storDb into dataDb.
-func (self *ApierV1) LoadAccountActions(attrs utils.TPAccountActions, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "Account", "Direction"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	dbReader := engine.NewDbReader(self.StorDb, self.RatingDb, self.AccountDb, attrs.TPid)
-
-	if _, err := engine.AccLock.Guard(attrs.KeyId(), func() (float64, error) {
-		if err := dbReader.LoadAccountActionsFiltered(&attrs); err != nil {
-			return 0, err
-		}
-		return 0, nil
-	}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	// ToDo: Get the action keys loaded by dbReader so we reload only these in cache
-	// Need to do it before scheduler otherwise actions to run will be unknown
-	if err := self.AccountDb.CacheAccounting(nil, nil, nil, []string{}); err != nil {
-		return err
-	}
-	if self.Sched != nil {
-		self.Sched.LoadActionTimings(self.AccountDb)
-		self.Sched.Restart()
-	}
-	*reply = OK
-	return nil
-}
-
-func (self *ApierV1) ReloadScheduler(input string, reply *string) error {
-	if self.Sched == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
-	}
-	self.Sched.LoadActionTimings(self.AccountDb)
-	self.Sched.Restart()
-	*reply = OK
-	return nil
-
-}
-
-func (self *ApierV1) ReloadCache(attrs utils.ApiReloadCache, reply *string) error {
-	var dstKeys, rpKeys, rpfKeys, actKeys, shgKeys, rpAlsKeys, accAlsKeys, lcrKeys, dcsKeys []string
-	if len(attrs.DestinationIds) > 0 {
-		dstKeys = make([]string, len(attrs.DestinationIds))
-		for idx, dId := range attrs.DestinationIds {
-			dstKeys[idx] = engine.DESTINATION_PREFIX + dId // Cache expects them as redis keys
-		}
-	}
-	if len(attrs.RatingPlanIds) > 0 {
-		rpKeys = make([]string, len(attrs.RatingPlanIds))
-		for idx, rpId := range attrs.RatingPlanIds {
-			rpKeys[idx] = engine.RATING_PLAN_PREFIX + rpId
-		}
-	}
-	if len(attrs.RatingProfileIds) > 0 {
-		rpfKeys = make([]string, len(attrs.RatingProfileIds))
-		for idx, rpfId := range attrs.RatingProfileIds {
-			rpfKeys[idx] = engine.RATING_PROFILE_PREFIX + rpfId
-		}
-	}
-	if len(attrs.ActionIds) > 0 {
-		actKeys = make([]string, len(attrs.ActionIds))
-		for idx, actId := range attrs.ActionIds {
-			actKeys[idx] = engine.ACTION_PREFIX + actId
-		}
-	}
-	if len(attrs.SharedGroupIds) > 0 {
-		shgKeys = make([]string, len(attrs.SharedGroupIds))
-		for idx, shgId := range attrs.SharedGroupIds {
-			shgKeys[idx] = engine.SHARED_GROUP_PREFIX + shgId
-		}
-	}
-	if len(attrs.RpAliases) > 0 {
-		rpAlsKeys = make([]string, len(attrs.RpAliases))
-		for idx, alias := range attrs.RpAliases {
-			rpAlsKeys[idx] = engine.RP_ALIAS_PREFIX + alias
-		}
-	}
-	if len(attrs.AccAliases) > 0 {
-		accAlsKeys = make([]string, len(attrs.AccAliases))
-		for idx, alias := range attrs.AccAliases {
-			accAlsKeys[idx] = engine.ACC_ALIAS_PREFIX + alias
-		}
-	}
-	if len(attrs.LCRIds) > 0 {
-		lcrKeys = make([]string, len(attrs.LCRIds))
-		for idx, lcrId := range attrs.LCRIds {
-			lcrKeys[idx] = engine.LCR_PREFIX + lcrId
-		}
-	}
-
-	if len(attrs.DerivedChargers) > 0 {
-		dcsKeys = make([]string, len(attrs.DerivedChargers))
-		for idx, dc := range attrs.DerivedChargers {
-			dcsKeys[idx] = engine.DERIVEDCHARGERS_PREFIX + dc
-		}
-	}
-	if err := self.RatingDb.CacheRating(dstKeys, rpKeys, rpfKeys, rpAlsKeys, lcrKeys); err != nil {
-		return err
-	}
-	if err := self.AccountDb.CacheAccounting(actKeys, shgKeys, accAlsKeys, dcsKeys); err != nil {
-		return err
-	}
-	*reply = "OK"
-	return nil
-}
-
-func (self *ApierV1) GetCacheStats(attrs utils.AttrCacheStats, reply *utils.CacheStats) error {
-	cs := new(utils.CacheStats)
-	cs.Destinations = cache2go.CountEntries(engine.DESTINATION_PREFIX)
-	cs.RatingPlans = cache2go.CountEntries(engine.RATING_PLAN_PREFIX)
-	cs.RatingProfiles = cache2go.CountEntries(engine.RATING_PROFILE_PREFIX)
-	cs.Actions = cache2go.CountEntries(engine.ACTION_PREFIX)
-	cs.SharedGroups = cache2go.CountEntries(engine.SHARED_GROUP_PREFIX)
-	cs.RatingAliases = cache2go.CountEntries(engine.RP_ALIAS_PREFIX)
-	cs.AccountAliases = cache2go.CountEntries(engine.ACC_ALIAS_PREFIX)
-	cs.DerivedChargers = cache2go.CountEntries(engine.DERIVEDCHARGERS_PREFIX)
-	*reply = *cs
-	return nil
-}
-
-func (self *ApierV1) GetCachedItemAge(itemId string, reply *utils.CachedItemAge) error {
-	if len(itemId) == 0 {
-		return fmt.Errorf("%s:ItemId", utils.ERR_MANDATORY_IE_MISSING)
-	}
-	cachedItemAge := new(utils.CachedItemAge)
-	var found bool
-	for idx, cacheKey := range []string{engine.DESTINATION_PREFIX + itemId, engine.RATING_PLAN_PREFIX + itemId, engine.RATING_PROFILE_PREFIX + itemId,
-		engine.ACTION_PREFIX + itemId, engine.SHARED_GROUP_PREFIX + itemId, engine.RP_ALIAS_PREFIX + itemId, engine.ACC_ALIAS_PREFIX + itemId} {
-		if age, err := cache2go.GetKeyAge(cacheKey); err == nil {
-			found = true
-			switch idx {
-			case 0:
-				cachedItemAge.Destination = age
-			case 1:
-				cachedItemAge.RatingPlan = age
-			case 2:
-				cachedItemAge.RatingProfile = age
-			case 3:
-				cachedItemAge.Action = age
-			case 4:
-				cachedItemAge.SharedGroup = age
-			case 5:
-				cachedItemAge.RatingAlias = age
-			case 6:
-				cachedItemAge.AccountAlias = age
-			}
-		}
-	}
-	if !found {
-		return errors.New(utils.ERR_NOT_FOUND)
-	}
-	*reply = *cachedItemAge
-	return nil
-}
-
-func (self *ApierV1) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder, reply *string) error {
-	loader := engine.NewFileCSVReader(self.RatingDb, self.AccountDb, utils.CSV_SEP,
-		path.Join(attrs.FolderPath, utils.DESTINATIONS_CSV),
-		path.Join(attrs.FolderPath, utils.TIMINGS_CSV),
-		path.Join(attrs.FolderPath, utils.RATES_CSV),
-		path.Join(attrs.FolderPath, utils.DESTINATION_RATES_CSV),
-		path.Join(attrs.FolderPath, utils.RATING_PLANS_CSV),
-		path.Join(attrs.FolderPath, utils.RATING_PROFILES_CSV),
-		path.Join(attrs.FolderPath, utils.SHARED_GROUPS_CSV),
-		path.Join(attrs.FolderPath, utils.LCRS_CSV),
-		path.Join(attrs.FolderPath, utils.ACTIONS_CSV),
-		path.Join(attrs.FolderPath, utils.ACTION_PLANS_CSV),
-		path.Join(attrs.FolderPath, utils.ACTION_TRIGGERS_CSV),
-		path.Join(attrs.FolderPath, utils.ACCOUNT_ACTIONS_CSV),
-		path.Join(attrs.FolderPath, utils.DERIVED_CHARGERS_CSV),
-		path.Join(attrs.FolderPath, utils.CDR_STATS_CSV))
-	if err := loader.LoadAll(); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	if attrs.DryRun {
-		*reply = "OK"
-		return nil // Mission complete, no errors
-	}
-	if err := loader.WriteToDatabase(attrs.FlushDb, false); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	// Make sure the items are in the cache
-	dstIds, _ := loader.GetLoadedIds(engine.DESTINATION_PREFIX)
-	dstKeys := make([]string, len(dstIds))
-	for idx, dId := range dstIds {
-		dstKeys[idx] = engine.DESTINATION_PREFIX + dId // Cache expects them as redis keys
-	}
-	rplIds, _ := loader.GetLoadedIds(engine.RATING_PLAN_PREFIX)
-	rpKeys := make([]string, len(rplIds))
-	for idx, rpId := range rplIds {
-		rpKeys[idx] = engine.RATING_PLAN_PREFIX + rpId
-	}
-	rpfIds, _ := loader.GetLoadedIds(engine.RATING_PROFILE_PREFIX)
-	rpfKeys := make([]string, len(rpfIds))
-	for idx, rpfId := range rpfIds {
-		rpfKeys[idx] = engine.RATING_PROFILE_PREFIX + rpfId
-	}
-	actIds, _ := loader.GetLoadedIds(engine.ACTION_PREFIX)
-	actKeys := make([]string, len(actIds))
-	for idx, actId := range actIds {
-		actKeys[idx] = engine.ACTION_PREFIX + actId
-	}
-	shgIds, _ := loader.GetLoadedIds(engine.SHARED_GROUP_PREFIX)
-	shgKeys := make([]string, len(shgIds))
-	for idx, shgId := range shgIds {
-		shgKeys[idx] = engine.SHARED_GROUP_PREFIX + shgId
-	}
-	rpAliases, _ := loader.GetLoadedIds(engine.RP_ALIAS_PREFIX)
-	rpAlsKeys := make([]string, len(rpAliases))
-	for idx, alias := range rpAliases {
-		rpAlsKeys[idx] = engine.RP_ALIAS_PREFIX + alias
-	}
-	accAliases, _ := loader.GetLoadedIds(engine.ACC_ALIAS_PREFIX)
-	accAlsKeys := make([]string, len(accAliases))
-	for idx, alias := range accAliases {
-		accAlsKeys[idx] = engine.ACC_ALIAS_PREFIX + alias
-	}
-	lcrIds, _ := loader.GetLoadedIds(engine.LCR_PREFIX)
-	lcrKeys := make([]string, len(lcrIds))
-	for idx, lcrId := range lcrIds {
-		lcrKeys[idx] = engine.LCR_PREFIX + lcrId
-	}
-	dcs, _ := loader.GetLoadedIds(engine.DERIVEDCHARGERS_PREFIX)
-	dcsKeys := make([]string, len(dcs))
-	for idx, dc := range dcs {
-		dcsKeys[idx] = engine.DERIVEDCHARGERS_PREFIX + dc
-	}
-	if err := self.RatingDb.CacheRating(dstKeys, rpKeys, rpfKeys, rpAlsKeys, lcrKeys); err != nil {
-		return err
-	}
-	if err := self.AccountDb.CacheAccounting(actKeys, shgKeys, accAlsKeys, dcsKeys); err != nil {
-		return err
-	}
-	if self.Sched != nil {
-		self.Sched.LoadActionTimings(self.AccountDb)
-		self.Sched.Restart()
-	}
-	cstKeys, _ := loader.GetLoadedIds(engine.CDR_STATS_PREFIX)
-	if len(cstKeys) != 0 && self.CdrStatsSrv != nil {
-		if err := self.CdrStatsSrv.ReloadQueues(cstKeys, nil); err != nil {
-			return err
-		}
-	}
-	*reply = "OK"
-	return nil
-}

apier/cdre.go

@@ -1,163 +0,0 @@
-/*
-Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package apier
-
-import (
-	"fmt"
-	"github.com/cgrates/cgrates/cdre"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
-	"path"
-	"strconv"
-	"strings"
-	"time"
-)
-
-// Export Cdrs to file
-func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.ExportedFileCdrs) error {
-	var tStart, tEnd time.Time
-	var err error
-	engine.Logger.Debug(fmt.Sprintf("ExportCdrsToFile: %+v", attr))
-	if len(attr.TimeStart) != 0 {
-		if tStart, err = utils.ParseTimeDetectLayout(attr.TimeStart); err != nil {
-			return err
-		}
-	}
-	if len(attr.TimeEnd) != 0 {
-		if tEnd, err = utils.ParseTimeDetectLayout(attr.TimeEnd); err != nil {
-			return err
-		}
-	}
-	exportTemplate := self.Config.CdreDefaultInstance
-	if attr.ExportTemplate != nil { // XML Template defined, can be field names or xml reference
-		if strings.HasPrefix(*attr.ExportTemplate, utils.XML_PROFILE_PREFIX) {
-			if self.Config.XmlCfgDocument == nil {
-				return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, "XmlDocumentNotLoaded")
-			}
-			expTplStr := *attr.ExportTemplate
-			if xmlTemplates := self.Config.XmlCfgDocument.GetCdreCfgs(expTplStr[len(utils.XML_PROFILE_PREFIX):]); xmlTemplates == nil {
-				return fmt.Errorf("%s:ExportTemplate", utils.ERR_NOT_FOUND)
-			} else {
-				exportTemplate = xmlTemplates[expTplStr[len(utils.XML_PROFILE_PREFIX):]].AsCdreConfig()
-			}
-		} else {
-			exportTemplate, _ = config.NewDefaultCdreConfig()
-			if contentFlds, err := config.NewCdreCdrFieldsFromIds(exportTemplate.CdrFormat == utils.CDRE_FIXED_WIDTH,
-				strings.Split(*attr.ExportTemplate, string(utils.CSV_SEP))...); err != nil {
-				return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-			} else {
-				exportTemplate.ContentFields = contentFlds
-			}
-		}
-	}
-	if exportTemplate == nil {
-		return fmt.Errorf("%s:ExportTemplate", utils.ERR_MANDATORY_IE_MISSING)
-	}
-	cdrFormat := exportTemplate.CdrFormat
-	if attr.CdrFormat != nil {
-		cdrFormat = strings.ToLower(*attr.CdrFormat)
-	}
-	if !utils.IsSliceMember(utils.CdreCdrFormats, cdrFormat) {
-		return fmt.Errorf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, "CdrFormat")
-	}
-	fieldSep := exportTemplate.FieldSeparator
-	if attr.FieldSeparator != nil {
-		fieldSep = *attr.FieldSeparator
-	}
-	exportDir := exportTemplate.ExportDir
-	if attr.ExportDir != nil {
-		exportDir = *attr.ExportDir
-	}
-	exportId := strconv.FormatInt(time.Now().Unix(), 10)
-	if attr.ExportId != nil {
-		exportId = *attr.ExportId
-	}
-	fileName := fmt.Sprintf("cdre_%s.%s", exportId, cdrFormat)
-	if attr.ExportFileName != nil {
-		fileName = *attr.ExportFileName
-	}
-	filePath := path.Join(exportDir, fileName)
-	if cdrFormat == utils.CDRE_DRYRUN {
-		filePath = utils.CDRE_DRYRUN
-	}
-	dataUsageMultiplyFactor := exportTemplate.DataUsageMultiplyFactor
-	if attr.DataUsageMultiplyFactor != nil {
-		dataUsageMultiplyFactor = *attr.DataUsageMultiplyFactor
-	}
-	costMultiplyFactor := exportTemplate.CostMultiplyFactor
-	if attr.CostMultiplyFactor != nil {
-		costMultiplyFactor = *attr.CostMultiplyFactor
-	}
-	costShiftDigits := exportTemplate.CostShiftDigits
-	if attr.CostShiftDigits != nil {
-		costShiftDigits = *attr.CostShiftDigits
-	}
-	roundingDecimals := exportTemplate.CostRoundingDecimals
-	if attr.RoundDecimals != nil {
-		roundingDecimals = *attr.RoundDecimals
-	}
-	maskDestId := exportTemplate.MaskDestId
-	if attr.MaskDestinationId != nil {
-		maskDestId = *attr.MaskDestinationId
-	}
-	maskLen := exportTemplate.MaskLength
-	if attr.MaskLength != nil {
-		maskLen = *attr.MaskLength
-	}
-	cdrs, err := self.CdrDb.GetStoredCdrs(attr.CgrIds, attr.MediationRunIds, attr.TORs, attr.CdrHosts, attr.CdrSources, attr.ReqTypes, attr.Directions,
-		attr.Tenants, attr.Categories, attr.Accounts, attr.Subjects, attr.DestinationPrefixes, attr.RatedAccounts, attr.RatedSubjects, attr.OrderIdStart, attr.OrderIdEnd,
-		tStart, tEnd, attr.SkipErrors, attr.SkipRated, false)
-	if err != nil {
-		return err
-	} else if len(cdrs) == 0 {
-		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
-		return nil
-	}
-	cdrexp, err := cdre.NewCdrExporter(cdrs, self.LogDb, exportTemplate, cdrFormat, fieldSep, exportId,
-		dataUsageMultiplyFactor, costMultiplyFactor, costShiftDigits, roundingDecimals, self.Config.RoundingDecimals, maskDestId, maskLen, self.Config.HttpSkipTlsVerify)
-	if err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	if cdrexp.TotalExportedCdrs() == 0 {
-		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
-		return nil
-	}
-	if err := cdrexp.WriteToFile(filePath); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	*reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), TotalCost: cdrexp.TotalCost(), FirstOrderId: cdrexp.FirstOrderId(), LastOrderId: cdrexp.LastOrderId()}
-	if !attr.SuppressCgrIds {
-		reply.ExportedCgrIds = cdrexp.PositiveExports()
-		reply.UnexportedCgrIds = cdrexp.NegativeExports()
-	}
-	return nil
-}
-
-// Remove Cdrs out of CDR storage
-func (self *ApierV1) RemCdrs(attrs utils.AttrRemCdrs, reply *string) error {
-	if len(attrs.CgrIds) == 0 {
-		return fmt.Errorf("%s:CgrIds", utils.ERR_MANDATORY_IE_MISSING)
-	}
-	if err := self.CdrDb.RemStoredCdrs(attrs.CgrIds); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	*reply = "OK"
-	return nil
-}

apier/cdrs.go

@@ -1,72 +0,0 @@
-/*
-Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
-
-This program is free software: you can Storagetribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITH*out ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package apier
-
-import (
-	"fmt"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
-	"time"
-)
-
-type AttrGetCallCost struct {
-	CgrId string // Unique id of the CDR
-	RunId string // Run Id
-}
-
-// Retrieves the callCost out of CGR logDb
-func (apier *ApierV1) GetCallCostLog(attrs AttrGetCallCost, reply *engine.CallCost) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"CgrId", "RunId"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if cc, err := apier.LogDb.GetCallCostLog(attrs.CgrId, "", attrs.RunId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if cc == nil {
-		return fmt.Errorf("NOT_FOUND")
-	} else {
-		*reply = *cc
-	}
-	return nil
-}
-
-// Retrieves CDRs based on the filters
-func (apier *ApierV1) GetCdrs(attrs utils.AttrGetCdrs, reply *[]*utils.CgrCdrOut) error {
-	var tStart, tEnd time.Time
-	var err error
-	if len(attrs.TimeStart) != 0 {
-		if tStart, err = utils.ParseTimeDetectLayout(attrs.TimeStart); err != nil {
-			return err
-		}
-	}
-	if len(attrs.TimeEnd) != 0 {
-		if tEnd, err = utils.ParseTimeDetectLayout(attrs.TimeEnd); err != nil {
-			return err
-		}
-	}
-	if cdrs, err := apier.CdrDb.GetStoredCdrs(attrs.CgrIds, attrs.MediationRunIds, attrs.TORs, attrs.CdrHosts, attrs.CdrSources, attrs.ReqTypes, attrs.Directions,
-		attrs.Tenants, attrs.Categories, attrs.Accounts, attrs.Subjects, attrs.DestinationPrefixes, attrs.RatedAccounts, attrs.RatedSubjects,
-		attrs.OrderIdStart, attrs.OrderIdEnd, tStart, tEnd, attrs.SkipErrors, attrs.SkipRated, false); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else {
-		for _, cdr := range cdrs {
-			*reply = append(*reply, cdr.AsCgrCdrOut())
-		}
-	}
-	return nil
-}

apier/tpaccountactions.go

@@ -1,104 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package apier
-
-import (
-	"errors"
-	"fmt"
-
-	"github.com/cgrates/cgrates/utils"
-)
-
-// Creates a new AccountActions profile within a tariff plan
-func (self *ApierV1) SetTPAccountActions(attrs utils.TPAccountActions, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs,
-		[]string{"TPid", "LoadId", "Tenant", "Account", "Direction", "ActionPlanId", "ActionTriggersId"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if err := self.StorDb.SetTPAccountActions(attrs.TPid, map[string]*utils.TPAccountActions{attrs.KeyId(): &attrs}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	*reply = "OK"
-	return nil
-}
-
-type AttrGetTPAccountActions struct {
-	TPid   string // Tariff plan id
-	LoadId string // AccountActions id
-
-}
-
-// Queries specific AccountActions profile on tariff plan
-func (self *ApierV1) GetTPAccountActions(attrs utils.TPAccountActions, reply *[]*utils.TPAccountActions) error {
-	mndtryFlds := []string{"TPid", "LoadId"}
-	if len(attrs.Account) != 0 { // If account provided as filter, make all related fields mandatory
-		mndtryFlds = append(mndtryFlds, "Tenant", "Account", "Direction")
-	}
-	if missing := utils.MissingStructFields(&attrs, mndtryFlds); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if aa, err := self.StorDb.GetTpAccountActions(&attrs); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if len(aa) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
-	} else {
-		var acts []*utils.TPAccountActions
-		if len(attrs.Account) != 0 {
-			acts = []*utils.TPAccountActions{aa[attrs.KeyId()]}
-		} else {
-			for _, actLst := range aa {
-				acts = append(acts, actLst)
-			}
-		}
-		*reply = acts
-	}
-	return nil
-}
-
-type AttrGetTPAccountActionIds struct {
-	TPid string // Tariff plan id
-}
-
-// Queries AccountActions identities on specific tariff plan.
-func (self *ApierV1) GetTPAccountActionLoadIds(attrs AttrGetTPAccountActionIds, reply *[]string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_ACCOUNT_ACTIONS, "loadid", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
-	} else {
-		*reply = ids
-	}
-	return nil
-}
-
-// Removes specific AccountActions on Tariff plan
-func (self *ApierV1) RemTPAccountActions(attrs utils.TPAccountActions, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "Account", "Direction"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_ACCOUNT_ACTIONS, attrs.TPid, attrs.LoadId, attrs.Tenant, attrs.Account, attrs.Direction); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else {
-		*reply = "OK"
-	}
-	return nil
-}

apier/tpratingprofiles.go

@@ -1,105 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package apier
-
-// This file deals with tp_rate_profiles management over APIs
-
-import (
-	"errors"
-	"fmt"
-
-	"github.com/cgrates/cgrates/utils"
-)
-
-// Creates a new RatingProfile within a tariff plan
-func (self *ApierV1) SetTPRatingProfile(attrs utils.TPRatingProfile, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "Category", "Direction", "Subject", "RatingPlanActivations"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if err := self.StorDb.SetTPRatingProfiles(attrs.TPid, map[string]*utils.TPRatingProfile{attrs.KeyId(): &attrs}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	*reply = "OK"
-	return nil
-}
-
-type AttrGetTPRatingProfile struct {
-	TPid   string // Tariff plan id
-	LoadId string // RatingProfile id
-}
-
-// Queries specific RatingProfile on tariff plan
-func (self *ApierV1) GetTPRatingProfiles(attrs utils.TPRatingProfile, reply *[]*utils.TPRatingProfile) error {
-	mndtryFlds := []string{"TPid", "LoadId"}
-	if len(attrs.Subject) != 0 { // If Subject provided as filter, make all related fields mandatory
-		mndtryFlds = append(mndtryFlds, "Tenant", "TOR", "Direction", "Subject")
-	}
-	if missing := utils.MissingStructFields(&attrs, mndtryFlds); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if dr, err := self.StorDb.GetTpRatingProfiles(&attrs); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if dr == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
-	} else {
-		var rpfs []*utils.TPRatingProfile
-		if len(attrs.Subject) != 0 {
-			rpfs = []*utils.TPRatingProfile{dr[attrs.KeyId()]}
-		} else {
-			for _, rpfLst := range dr {
-				rpfs = append(rpfs, rpfLst)
-			}
-		}
-		*reply = rpfs
-	}
-	return nil
-}
-
-// Queries RatingProfile identities on specific tariff plan.
-func (self *ApierV1) GetTPRatingProfileLoadIds(attrs utils.AttrTPRatingProfileIds, reply *[]string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_RATE_PROFILES, "loadid", map[string]string{
-		"tenant":    attrs.Tenant,
-		"tor":       attrs.Category,
-		"direction": attrs.Direction,
-		"subject":   attrs.Subject,
-	}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
-	} else {
-		*reply = ids
-	}
-	return nil
-}
-
-// Removes specific RatingProfile on Tariff plan
-func (self *ApierV1) RemTPRatingProfile(attrs utils.TPRatingProfile, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "TOR", "Direction", "Subject"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_RATE_PROFILES, attrs.TPid, attrs.LoadId, attrs.Tenant, attrs.Category, attrs.Direction, attrs.Subject); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else {
-		*reply = "OK"
-	}
-	return nil
-}

apier/tutfscsv_local_test.go

@@ -1,285 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package apier
-
-import (
-	"fmt"
-	"net/rpc/jsonrpc"
-	"os"
-	"os/exec"
-	"path"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
-)
-
-var fscsvCfgPath string
-var fscsvCfg *config.CGRConfig
-
-func init() {
-	fscsvCfgPath = path.Join(*dataDir, "tutorials", "fs_csv", "cgrates", "etc", "cgrates", "cgrates.cfg")
-	fscsvCfg, _ = config.NewCGRConfigFromFile(&fscsvCfgPath)
-}
-
-// Remove here so they can be properly created by init script
-func TestFsCsvRemoveDirs(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	for _, pathDir := range []string{cfg.CdreDefaultInstance.ExportDir, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.HistoryDir} {
-		if err := os.RemoveAll(pathDir); err != nil {
-			t.Fatal("Error removing folder: ", pathDir, err)
-		}
-	}
-}
-
-// Empty tables before using them
-func TestFsCsvCreateTables(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	if *storDbType != utils.MYSQL {
-		t.Fatal("Unsupported storDbType")
-	}
-	var mysql *engine.MySQLStorage
-	if d, err := engine.NewMySQLStorage(fscsvCfg.StorDBHost, fscsvCfg.StorDBPort, fscsvCfg.StorDBName, fscsvCfg.StorDBUser, fscsvCfg.StorDBPass); err != nil {
-		t.Fatal("Error on opening database connection: ", err)
-	} else {
-		mysql = d.(*engine.MySQLStorage)
-	}
-	for _, scriptName := range []string{engine.CREATE_CDRS_TABLES_SQL, engine.CREATE_TARIFFPLAN_TABLES_SQL} {
-		if err := mysql.CreateTablesFromScript(path.Join(*dataDir, "storage", *storDbType, scriptName)); err != nil {
-			t.Fatal("Error on mysql creation: ", err.Error())
-			return // No point in going further
-		}
-	}
-	for _, tbl := range []string{utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_EXTRA} {
-		if _, err := mysql.Db.Query(fmt.Sprintf("SELECT 1 from %s", tbl)); err != nil {
-			t.Fatal(err.Error())
-		}
-	}
-}
-
-func TestFsCsvInitDataDb(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	ratingDb, err := engine.ConfigureRatingStorage(fscsvCfg.RatingDBType, fscsvCfg.RatingDBHost, fscsvCfg.RatingDBPort, fscsvCfg.RatingDBName, fscsvCfg.RatingDBUser, fscsvCfg.RatingDBPass, fscsvCfg.DBDataEncoding)
-	if err != nil {
-		t.Fatal("Cannot connect to dataDb", err)
-	}
-	accountDb, err := engine.ConfigureAccountingStorage(fscsvCfg.AccountDBType, fscsvCfg.AccountDBHost, fscsvCfg.AccountDBPort, fscsvCfg.AccountDBName,
-		fscsvCfg.AccountDBUser, fscsvCfg.AccountDBPass, fscsvCfg.DBDataEncoding)
-	if err != nil {
-		t.Fatal("Cannot connect to dataDb", err)
-	}
-	for _, db := range []engine.Storage{ratingDb, accountDb} {
-		if err := db.Flush(); err != nil {
-			t.Fatal("Cannot reset dataDb", err)
-		}
-	}
-}
-
-func TestFsCsvStartFs(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	exec.Command("pkill", "freeswitch").Run() // Just to make sure no freeswitch is running
-	go func() {
-		fs := exec.Command("sudo", "/usr/share/cgrates/tutorials/fs_csv/freeswitch/etc/init.d/freeswitch", "start")
-		out, _ := fs.CombinedOutput()
-		engine.Logger.Info(fmt.Sprintf("CgrEngine-TestFsCsv: %s", out))
-	}()
-	time.Sleep(time.Duration(*waitFs) * time.Millisecond) // Give time to rater to fire up
-}
-
-// Finds cgr-engine executable and starts it with default configuration
-func TestFsCsvStartEngine(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	exec.Command("pkill", "cgr-engine").Run() // Just to make sure another one is not running, bit brutal maybe we can fine tune it
-	go func() {
-		eng := exec.Command("sudo", "/usr/share/cgrates/tutorials/fs_json/cgrates/etc/init.d/cgrates", "start")
-		out, _ := eng.CombinedOutput()
-		engine.Logger.Info(fmt.Sprintf("CgrEngine-TestFsCsv: %s", out))
-	}()
-	time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time to rater to fire up
-}
-
-// Connect rpc client to rater
-func TestFsCsvRpcConn(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	var err error
-	rater, err = jsonrpc.Dial("tcp", fscsvCfg.RPCJSONListen)
-	if err != nil {
-		t.Fatal("Could not connect to rater: ", err.Error())
-	}
-}
-
-// Make sure we start with fresh data
-func TestFsCsvEmptyCache(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	var rcvStats *utils.CacheStats
-	expectedStats := &utils.CacheStats{Destinations: 0, RatingPlans: 0, RatingProfiles: 0, Actions: 0}
-	var args utils.AttrCacheStats
-	if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil {
-		t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())
-	} else if !reflect.DeepEqual(expectedStats, rcvStats) {
-		t.Errorf("Calling ApierV1.GetCacheStats expected: %v, received: %v", expectedStats, rcvStats)
-	}
-}
-
-func TestFsCsvLoadTariffPlans(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	reply := ""
-	// Simple test that command is executed without errors
-	attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tutorials", "fs_csv", "cgrates", "tariffplans")}
-	if err := rater.Call("ApierV1.LoadTariffPlanFromFolder", attrs, &reply); err != nil {
-		t.Error("Got error on ApierV1.LoadTariffPlanFromFolder: ", err.Error())
-	} else if reply != "OK" {
-		t.Error("Calling ApierV1.LoadTariffPlanFromFolder got reply: ", reply)
-	}
-	time.Sleep(100 * time.Millisecond) // Give time for scheduler to execute topups
-	var rcvStats *utils.CacheStats
-	expectedStats := &utils.CacheStats{Destinations: 3, RatingPlans: 1, RatingProfiles: 1, Actions: 2}
-	var args utils.AttrCacheStats
-	if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil {
-		t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())
-	} else if !reflect.DeepEqual(expectedStats, rcvStats) {
-		t.Errorf("Calling ApierV1.GetCacheStats expected: %v, received: %v", expectedStats, rcvStats)
-	}
-}
-
-func TestFsCsvGetAccount(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	var reply *engine.Account
-	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001", Direction: "*out"}
-	if err := rater.Call("ApierV1.GetAccount", attrs, &reply); err != nil {
-		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
-	} else if reply.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 10.0 { // We expect 11.5 since we have added in the previous test 1.5
-		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", reply.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
-	}
-}
-
-func TestFsCsvCall1(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	tStart := time.Date(2014, 01, 15, 6, 0, 0, 0, time.UTC)
-	tEnd := time.Date(2014, 01, 15, 6, 0, 35, 0, time.UTC)
-	cd := engine.CallDescriptor{
-		Direction:     "*out",
-		Category:      "call",
-		Tenant:        "cgrates.org",
-		Subject:       "1001",
-		Account:       "1001",
-		Destination:   "1002",
-		TimeStart:     tStart,
-		TimeEnd:       tEnd,
-		DurationIndex: 35,
-	}
-	var cc engine.CallCost
-	// Make sure the cost is what we expect it is
-	if err := rater.Call("Responder.GetCost", cd, &cc); err != nil {
-		t.Error("Got error on Responder.GetCost: ", err.Error())
-	} else if cc.GetConnectFee() != 0.4 && cc.Cost != 0.6 {
-		t.Errorf("Calling Responder.GetCost got callcost: %v", cc)
-	}
-	// Make sure debit charges what cost returned
-	if err := rater.Call("Responder.MaxDebit", cd, &cc); err != nil {
-		t.Error("Got error on Responder.MaxDebit: ", err.Error())
-	} else if cc.GetConnectFee() != 0.4 && cc.Cost != 0.6 {
-		t.Errorf("Calling Responder.MaxDebit got callcost: %v", cc)
-	}
-	// Make sure the account was debited correctly for the first loop index (ConnectFee included)
-	var reply *engine.Account
-	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001", Direction: "*out"}
-	if err := rater.Call("ApierV1.GetAccount", attrs, &reply); err != nil {
-		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
-	} else if reply.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 9.4 { // We expect 11.5 since we have added in the previous test 1.5
-		t.Errorf("Calling ApierV1.GetAccount expected: 9.4, received: %f", reply.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
-	} else if len(reply.UnitCounters) != 1 ||
-		utils.Round(reply.UnitCounters[0].Balances[0].Value, 2, utils.ROUNDING_MIDDLE) != 0.6 { // Make sure we correctly count usage
-		t.Errorf("Received unexpected UnitCounters: %v", reply.UnitCounters)
-	}
-	cd = engine.CallDescriptor{
-		Direction:     "*out",
-		Category:      "call",
-		Tenant:        "cgrates.org",
-		Subject:       "1001",
-		Account:       "1001",
-		Destination:   "1002",
-		TimeStart:     tStart,
-		TimeEnd:       tEnd,
-		DurationIndex: 35,
-		LoopIndex:     1, // Should not charge ConnectFee
-	}
-	// Make sure debit charges what cost returned
-	if err := rater.Call("Responder.MaxDebit", cd, &cc); err != nil {
-		t.Error("Got error on Responder.MaxDebit: ", err.Error())
-	} else if cc.GetConnectFee() != 0.4 && cc.Cost != 0.2 { // Does not contain connectFee, however connectFee should be correctly reported
-		t.Errorf("Calling Responder.MaxDebit got callcost: %v", cc)
-	}
-	// Make sure the account was debited correctly for the first loop index (ConnectFee included)
-	var reply2 *engine.Account
-	if err := rater.Call("ApierV1.GetAccount", attrs, &reply2); err != nil {
-		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
-	} else if utils.Round(reply2.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue(), 2, utils.ROUNDING_MIDDLE) != 9.20 {
-		t.Errorf("Calling ApierV1.GetAccount expected: 9.2, received: %f", reply2.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
-	} else if len(reply2.UnitCounters) != 1 ||
-		utils.Round(reply2.UnitCounters[0].Balances[0].Value, 2, utils.ROUNDING_MIDDLE) != 0.8 { // Make sure we correctly count usage
-		t.Errorf("Received unexpected UnitCounters: %v", reply2.UnitCounters)
-	}
-}
-
-// Simply kill the engine after we are done with tests within this file
-func TestFsCsvStopEngine(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	go func() {
-		eng := exec.Command("/usr/share/cgrates/tutorials/fs_csv/cgrates/etc/init.d/cgrates", "stop")
-		out, _ := eng.CombinedOutput()
-		engine.Logger.Info(fmt.Sprintf("CgrEngine-TestFsCsv: %s", out))
-	}()
-}
-
-func TestFsCsvStopFs(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	go func() {
-		fs := exec.Command("/usr/share/cgrates/tutorials/fs_csv/freeswitch/etc/init.d/freeswitch", "stop")
-		out, _ := fs.CombinedOutput()
-		engine.Logger.Info(fmt.Sprintf("CgrEngine-TestFsCsv: %s", out))
-	}()
-}

apier/v1/accounts.go

@@ -1,6 +1,6 @@
 /*
 Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,11 +16,12 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"errors"
-	"fmt"
+	"math"
+	"regexp"
+	"strings"
 	"time"
 
 	"github.com/cgrates/cgrates/engine"
@@ -42,12 +43,12 @@ type AccountActionTiming struct {
 
 func (self *ApierV1) GetAccountActionPlan(attrs AttrAcntAction, reply *[]*AccountActionTiming) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Direction"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(strings.Join(missing, ","), "")
 	}
 	accountATs := make([]*AccountActionTiming, 0)
-	allATs, err := self.AccountDb.GetAllActionTimings()
+	allATs, err := self.RatingDb.GetAllActionPlans()
 	if err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	}
 	for _, ats := range allATs {
 		for _, at := range ats {
@@ -72,31 +73,31 @@ type AttrRemActionTiming struct {
 // Removes an ActionTimings or parts of it depending on filters being set
 func (self *ApierV1) RemActionTiming(attrs AttrRemActionTiming, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"ActionPlanId"}); len(missing) != 0 { // Only mandatory ActionPlanId
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	if len(attrs.Account) != 0 { // Presence of Account requires complete account details to be provided
 		if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Direction"}); len(missing) != 0 {
-			return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+			return utils.NewErrMandatoryIeMissing(missing...)
 		}
 	}
-	_, err := engine.AccLock.Guard(engine.ACTION_TIMING_PREFIX, func() (float64, error) {
-		ats, err := self.AccountDb.GetActionTimings(attrs.ActionPlanId)
+	_, err := engine.Guardian.Guard(func() (interface{}, error) {
+		ats, err := self.RatingDb.GetActionPlans(attrs.ActionPlanId)
 		if err != nil {
 			return 0, err
 		} else if len(ats) == 0 {
-			return 0, errors.New(utils.ERR_NOT_FOUND)
+			return 0, utils.ErrNotFound
 		}
-		ats = engine.RemActionTiming(ats, attrs.ActionTimingId, utils.AccountKey(attrs.Tenant, attrs.Account, attrs.Direction))
-		if err := self.AccountDb.SetActionTimings(attrs.ActionPlanId, ats); err != nil {
+		ats = engine.RemActionPlan(ats, attrs.ActionTimingId, utils.AccountKey(attrs.Tenant, attrs.Account, attrs.Direction))
+		if err := self.RatingDb.SetActionPlans(attrs.ActionPlanId, ats); err != nil {
 			return 0, err
 		}
 		return 0, nil
-	})
+	}, utils.ACTION_TIMING_PREFIX)
 	if err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	}
 	if attrs.ReloadScheduler && self.Sched != nil {
-		self.Sched.LoadActionTimings(self.AccountDb)
+		self.Sched.LoadActionPlans(self.RatingDb)
 		self.Sched.Restart()
 	}
 	*reply = OK
@@ -106,10 +107,10 @@ func (self *ApierV1) RemActionTiming(attrs AttrRemActionTiming, reply *string) e
 // Returns a list of ActionTriggers on an account
 func (self *ApierV1) GetAccountActionTriggers(attrs AttrAcntAction, reply *engine.ActionTriggerPriotityList) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Direction"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	if balance, err := self.AccountDb.GetAccount(utils.AccountKey(attrs.Tenant, attrs.Account, attrs.Direction)); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = balance.ActionTriggers
 	}
@@ -117,62 +118,52 @@ func (self *ApierV1) GetAccountActionTriggers(attrs AttrAcntAction, reply *engin
 }
 
 type AttrRemAcntActionTriggers struct {
-	Tenant          string // Tenant he account belongs to
-	Account         string // Account name
-	Direction       string // Traffic direction
-	ActionTriggerId string // Id filtering only specific id to remove
+	Tenant           string // Tenant he account belongs to
+	Account          string // Account name
+	Direction        string // Traffic direction
+	ActionTriggersId string // Id filtering only specific id to remove (can be regexp pattern)
 }
 
 // Returns a list of ActionTriggers on an account
 func (self *ApierV1) RemAccountActionTriggers(attrs AttrRemAcntActionTriggers, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Direction"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	balanceId := utils.AccountKey(attrs.Tenant, attrs.Account, attrs.Direction)
-	_, err := engine.AccLock.Guard(balanceId, func() (float64, error) {
+	_, err := engine.Guardian.Guard(func() (interface{}, error) {
 		ub, err := self.AccountDb.GetAccount(balanceId)
 		if err != nil {
 			return 0, err
 		}
-		for idx, actr := range ub.ActionTriggers {
-			if len(attrs.ActionTriggerId) != 0 && actr.Id != attrs.ActionTriggerId { // Empty actionTriggerId will match always
-				continue
-			}
-			if len(ub.ActionTriggers) != 1 { // Remove by index
-				ub.ActionTriggers[idx], ub.ActionTriggers = ub.ActionTriggers[len(ub.ActionTriggers)-1], ub.ActionTriggers[:len(ub.ActionTriggers)-1]
-			} else { // For last item, simply reinit the slice
-				ub.ActionTriggers = make(engine.ActionTriggerPriotityList, 0)
+		nactrs := make(engine.ActionTriggerPriotityList, 0)
+		for _, actr := range ub.ActionTriggers {
+			match, _ := regexp.MatchString(attrs.ActionTriggersId, actr.Id)
+			if len(attrs.ActionTriggersId) != 0 && !match {
+				nactrs = append(nactrs, actr)
 			}
 		}
+		ub.ActionTriggers = nactrs
 		if err := self.AccountDb.SetAccount(ub); err != nil {
 			return 0, err
 		}
 		return 0, nil
-	})
+	}, balanceId)
 	if err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	}
 	*reply = OK
 	return nil
 }
 
-type AttrSetAccount struct {
-	Tenant        string
-	Direction     string
-	Account       string
-	ActionPlanId  string
-	AllowNegative bool
-}
-
 // Ads a new account into dataDb. If already defined, returns success.
-func (self *ApierV1) SetAccount(attr AttrSetAccount, reply *string) error {
+func (self *ApierV1) SetAccount(attr utils.AttrSetAccount, reply *string) error {
 	if missing := utils.MissingStructFields(&attr, []string{"Tenant", "Direction", "Account"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	balanceId := utils.AccountKey(attr.Tenant, attr.Account, attr.Direction)
 	var ub *engine.Account
-	var ats engine.ActionPlan
-	_, err := engine.AccLock.Guard(balanceId, func() (float64, error) {
+	var ats engine.ActionPlans
+	_, err := engine.Guardian.Guard(func() (interface{}, error) {
 		if bal, _ := self.AccountDb.GetAccount(balanceId); bal != nil {
 			ub = bal
 		} else { // Not found in db, create it here
@@ -184,7 +175,7 @@ func (self *ApierV1) SetAccount(attr AttrSetAccount, reply *string) error {
 
 		if len(attr.ActionPlanId) != 0 {
 			var err error
-			ats, err = self.AccountDb.GetActionTimings(attr.ActionPlanId)
+			ats, err = self.RatingDb.GetActionPlans(attr.ActionPlanId)
 			if err != nil {
 				return 0, err
 			}
@@ -197,25 +188,87 @@ func (self *ApierV1) SetAccount(attr AttrSetAccount, reply *string) error {
 			return 0, err
 		}
 		return 0, nil
-	})
+	}, balanceId)
 	if err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	}
 	if len(ats) != 0 {
-		_, err := engine.AccLock.Guard(engine.ACTION_TIMING_PREFIX, func() (float64, error) { // ToDo: Try locking it above on read somehow
-			if err := self.AccountDb.SetActionTimings(attr.ActionPlanId, ats); err != nil {
+		_, err := engine.Guardian.Guard(func() (interface{}, error) { // ToDo: Try locking it above on read somehow
+			if err := self.RatingDb.SetActionPlans(attr.ActionPlanId, ats); err != nil {
 				return 0, err
 			}
 			return 0, nil
-		})
+		}, utils.ACTION_TIMING_PREFIX)
 		if err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+			return utils.NewErrServerError(err)
 		}
 		if self.Sched != nil {
-			self.Sched.LoadActionTimings(self.AccountDb)
+			self.Sched.LoadActionPlans(self.RatingDb)
 			self.Sched.Restart()
 		}
 	}
 	*reply = OK // This will mark saving of the account, error still can show up in actionTimingsId
 	return nil
 }
+
+func (self *ApierV1) RemoveAccount(attr utils.AttrRemoveAccount, reply *string) error {
+	if missing := utils.MissingStructFields(&attr, []string{"Tenant", "Direction", "Account"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	accountId := utils.AccountKey(attr.Tenant, attr.Account, attr.Direction)
+	_, err := engine.Guardian.Guard(func() (interface{}, error) {
+		if err := self.AccountDb.RemoveAccount(accountId); err != nil {
+			return 0, err
+		}
+		return 0, nil
+	}, accountId)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+
+	*reply = OK
+	return nil
+}
+
+func (self *ApierV1) GetAccounts(attr utils.AttrGetAccounts, reply *[]*engine.Account) error {
+	if len(attr.Tenant) == 0 {
+		return utils.NewErrMandatoryIeMissing("Tenanat")
+	}
+	if len(attr.Direction) == 0 {
+		attr.Direction = utils.OUT
+	}
+	var accountKeys []string
+	var err error
+	if len(attr.AccountIds) == 0 {
+		if accountKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACCOUNT_PREFIX + utils.ConcatenatedKey(attr.Direction, attr.Tenant)); err != nil {
+			return err
+		}
+	} else {
+		for _, acntId := range attr.AccountIds {
+			if len(acntId) == 0 { // Source of error returned from redis (key not found)
+				continue
+			}
+			accountKeys = append(accountKeys, utils.ACCOUNT_PREFIX+utils.ConcatenatedKey(attr.Direction, attr.Tenant, acntId))
+		}
+	}
+	if len(accountKeys) == 0 {
+		return nil
+	}
+	var limitedAccounts []string
+	if attr.Limit != 0 {
+		max := math.Min(float64(attr.Offset+attr.Limit), float64(len(accountKeys)))
+		limitedAccounts = accountKeys[attr.Offset:int(max)]
+	} else {
+		limitedAccounts = accountKeys[attr.Offset:]
+	}
+	retAccounts := make([]*engine.Account, 0)
+	for _, acntKey := range limitedAccounts {
+		if acnt, err := self.AccountDb.GetAccount(acntKey[len(utils.ACCOUNT_PREFIX):]); err != nil && err != utils.ErrNotFound { // Not found is not an error here
+			return err
+		} else if acnt != nil {
+			retAccounts = append(retAccounts, acnt)
+		}
+	}
+	*reply = retAccounts
+	return nil
+}

apier/v1/accounts_test.go

@@ -0,0 +1,106 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"testing"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var (
+	apierAcnts            *ApierV1
+	apierAcntsAcntStorage *engine.MapStorage
+)
+
+func init() {
+	apierAcntsAcntStorage, _ = engine.NewMapStorage()
+	cfg, _ := config.NewDefaultCGRConfig()
+	apierAcnts = &ApierV1{AccountDb: engine.AccountingStorage(apierAcntsAcntStorage), Config: cfg}
+}
+
+func TestSetAccounts(t *testing.T) {
+	cgrTenant := "cgrates.org"
+	iscTenant := "itsyscom.com"
+	b10 := &engine.Balance{Value: 10, Weight: 10}
+	cgrAcnt1 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, cgrTenant, "account1"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	cgrAcnt2 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, cgrTenant, "account2"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	cgrAcnt3 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, cgrTenant, "account3"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	iscAcnt1 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, iscTenant, "account1"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	iscAcnt2 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, iscTenant, "account2"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	for _, account := range []*engine.Account{cgrAcnt1, cgrAcnt2, cgrAcnt3, iscAcnt1, iscAcnt2} {
+		if err := apierAcntsAcntStorage.SetAccount(account); err != nil {
+			t.Error(err)
+		}
+	}
+	apierAcntsAcntStorage.CachePrefixes(utils.ACTION_PREFIX)
+}
+
+/*
+func TestGetAccountIds(t *testing.T) {
+	var accountIds []string
+	var attrs AttrGetAccountIds
+	if err := apierAcnts.GetAccountIds(attrs, &accountIds); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accountIds) != 5 {
+		t.Errorf("Accounts returned: %+v", accountIds)
+	}
+}
+*/
+
+func TestGetAccounts(t *testing.T) {
+	var accounts []*engine.Account
+	var attrs utils.AttrGetAccounts
+	if err := apierAcnts.GetAccounts(utils.AttrGetAccounts{Tenant: "cgrates.org"}, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 3 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+	attrs = utils.AttrGetAccounts{Tenant: "itsyscom.com"}
+	if err := apierAcnts.GetAccounts(attrs, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 2 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+	attrs = utils.AttrGetAccounts{Tenant: "cgrates.org", AccountIds: []string{"account1"}}
+	if err := apierAcnts.GetAccounts(attrs, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 1 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+	attrs = utils.AttrGetAccounts{Tenant: "itsyscom.com", AccountIds: []string{"INVALID"}}
+	if err := apierAcnts.GetAccounts(attrs, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 0 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+	attrs = utils.AttrGetAccounts{Tenant: "INVALID"}
+	if err := apierAcnts.GetAccounts(attrs, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 0 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+}

apier/v1/aliases.go

@@ -1,6 +1,6 @@
 /*
 Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,11 +16,9 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"errors"
-	"fmt"
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
@@ -38,18 +36,17 @@ type AttrAddAccountAliases struct {
 // Retrieve aliases configured for a rating profile subject
 func (self *ApierV1) AddRatingSubjectAliases(attrs AttrAddRatingSubjectAliases, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Subject", "Aliases"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	aliasesChanged := []string{}
 	for _, alias := range attrs.Aliases {
 		if err := self.RatingDb.SetRpAlias(utils.RatingSubjectAliasKey(attrs.Tenant, alias), attrs.Subject); err != nil {
-			return fmt.Errorf("%s:%s:%s", utils.ERR_SERVER_ERROR, alias, err.Error())
+			return utils.NewErrServerError(err)
 		}
-		aliasesChanged = append(aliasesChanged, engine.RP_ALIAS_PREFIX+utils.RatingSubjectAliasKey(attrs.Tenant, alias))
+		aliasesChanged = append(aliasesChanged, utils.RP_ALIAS_PREFIX+utils.RatingSubjectAliasKey(attrs.Tenant, alias))
 	}
-	didNotChange := []string{}
-	if err := self.RatingDb.CacheRating(didNotChange, didNotChange, didNotChange, aliasesChanged, didNotChange); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.RatingDb.CachePrefixValues(map[string][]string{utils.RP_ALIAS_PREFIX: aliasesChanged}); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = utils.OK
 	return nil
@@ -58,12 +55,12 @@ func (self *ApierV1) AddRatingSubjectAliases(attrs AttrAddRatingSubjectAliases,
 // Retrieve aliases configured for a rating profile subject
 func (self *ApierV1) GetRatingSubjectAliases(attrs engine.TenantRatingSubject, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Subject"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	if aliases, err := self.RatingDb.GetRPAliases(attrs.Tenant, attrs.Subject, false); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	} else if len(aliases) == 0 { // Need it since otherwise we get some unexpected errrors in the client
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = aliases
 	}
@@ -73,33 +70,36 @@ func (self *ApierV1) GetRatingSubjectAliases(attrs engine.TenantRatingSubject, r
 // Retrieve aliases configured for a rating profile subject
 func (self *ApierV1) RemRatingSubjectAliases(tenantRatingSubject engine.TenantRatingSubject, reply *string) error {
 	if missing := utils.MissingStructFields(&tenantRatingSubject, []string{"Tenant", "Subject"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.RatingDb.RemoveRpAliases([]*engine.TenantRatingSubject{&tenantRatingSubject}); err != nil {
-		return fmt.Errorf("%s:% s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	didNotChange := []string{}
-	if err := self.RatingDb.CacheRating(didNotChange, didNotChange, didNotChange, nil, didNotChange); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.RatingDb.RemoveRpAliases([]*engine.TenantRatingSubject{&tenantRatingSubject}, false); err != nil {
+		if err == utils.ErrNotFound {
+			return err
+		}
+		return utils.NewErrServerError(err)
 	}
+
+	// cache refresh not needed, synched in RemoveRpAliases
+	/*if err := self.RatingDb.CachePrefixes(utils.RP_ALIAS_PREFIX); err != nil {
+		return utils.NewErrServerError(err)
+	}*/
 	*reply = utils.OK
 	return nil
 }
 
 func (self *ApierV1) AddAccountAliases(attrs AttrAddAccountAliases, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Aliases"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	aliasesChanged := []string{}
 	for _, alias := range attrs.Aliases {
-		if err := self.AccountDb.SetAccAlias(utils.AccountAliasKey(attrs.Tenant, alias), attrs.Account); err != nil {
-			return fmt.Errorf("%s:%s:%s", utils.ERR_SERVER_ERROR, alias, err.Error())
+		if err := self.RatingDb.SetAccAlias(utils.AccountAliasKey(attrs.Tenant, alias), attrs.Account); err != nil {
+			return utils.NewErrServerError(err)
 		}
-		aliasesChanged = append(aliasesChanged, engine.ACC_ALIAS_PREFIX+utils.AccountAliasKey(attrs.Tenant, alias))
+		aliasesChanged = append(aliasesChanged, utils.ACC_ALIAS_PREFIX+utils.AccountAliasKey(attrs.Tenant, alias))
 	}
-	didNotChange := []string{}
-	if err := self.AccountDb.CacheAccounting(didNotChange, didNotChange, aliasesChanged, didNotChange); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.RatingDb.CachePrefixValues(map[string][]string{utils.ACC_ALIAS_PREFIX: aliasesChanged}); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = utils.OK
 	return nil
@@ -108,12 +108,12 @@ func (self *ApierV1) AddAccountAliases(attrs AttrAddAccountAliases, reply *strin
 // Retrieve aliases configured for an account
 func (self *ApierV1) GetAccountAliases(attrs engine.TenantAccount, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if aliases, err := self.AccountDb.GetAccountAliases(attrs.Tenant, attrs.Account, false); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if aliases, err := self.RatingDb.GetAccountAliases(attrs.Tenant, attrs.Account, false); err != nil {
+		return utils.NewErrServerError(err)
 	} else if len(aliases) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = aliases
 	}
@@ -123,15 +123,18 @@ func (self *ApierV1) GetAccountAliases(attrs engine.TenantAccount, reply *[]stri
 // Retrieve aliases configured for a rating profile subject
 func (self *ApierV1) RemAccountAliases(tenantAccount engine.TenantAccount, reply *string) error {
 	if missing := utils.MissingStructFields(&tenantAccount, []string{"Tenant", "Account"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.AccountDb.RemoveAccAliases([]*engine.TenantAccount{&tenantAccount}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	didNotChange := []string{}
-	if err := self.AccountDb.CacheAccounting(didNotChange, didNotChange, nil, didNotChange); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.RatingDb.RemoveAccAliases([]*engine.TenantAccount{&tenantAccount}, false); err != nil {
+		if err == utils.ErrNotFound {
+			return err
+		}
+		return utils.NewErrServerError(err)
 	}
+	// cache refresh not needed, synched in RemoveRpAliases
+	/*if err := self.RatingDb.CachePrefixes(utils.ACC_ALIAS_PREFIX); err != nil {
+		return utils.NewErrServerError(err)
+	}*/
 	*reply = utils.OK
 	return nil
 }

apier/v1/auth.go

@@ -0,0 +1,74 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Returns MaxUsage (for calls in seconds), -1 for no limit
+func (self *ApierV1) GetMaxUsage(usageRecord engine.UsageRecord, maxUsage *float64) error {
+	out, err := engine.LoadUserProfile(usageRecord, "")
+	if err != nil {
+		return err
+	}
+	usageRecord = out.(engine.UsageRecord)
+	if usageRecord.TOR == "" {
+		usageRecord.TOR = utils.VOICE
+	}
+	if usageRecord.ReqType == "" {
+		usageRecord.ReqType = self.Config.DefaultReqType
+	}
+	if usageRecord.Direction == "" {
+		usageRecord.Direction = utils.OUT
+	}
+	if usageRecord.Tenant == "" {
+		usageRecord.Tenant = self.Config.DefaultTenant
+	}
+	if usageRecord.Category == "" {
+		usageRecord.Category = self.Config.DefaultCategory
+	}
+	if usageRecord.Subject == "" {
+		usageRecord.Subject = usageRecord.Account
+	}
+	if usageRecord.SetupTime == "" {
+		usageRecord.SetupTime = utils.META_NOW
+	}
+	if usageRecord.Usage == "" {
+		usageRecord.Usage = strconv.FormatFloat(self.Config.MaxCallDuration.Seconds(), 'f', -1, 64)
+	}
+	storedCdr, err := usageRecord.AsStoredCdr()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	var maxDur float64
+	if err := self.Responder.GetDerivedMaxSessionTime(storedCdr, &maxDur); err != nil {
+		return err
+	}
+	if maxDur == -1.0 {
+		*maxUsage = -1.0
+		return nil
+	}
+	*maxUsage = time.Duration(maxDur).Seconds()
+	return nil
+}

apier/v1/cdre.go

@@ -0,0 +1,208 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/cgrates/cgrates/cdre"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func (self *ApierV1) ExportCdrsToZipString(attr utils.AttrExpFileCdrs, reply *string) error {
+	tmpDir := "/tmp"
+	attr.ExportDir = &tmpDir // Enforce exporting to tmp always so we avoid cleanup issues
+	efc := utils.ExportedFileCdrs{}
+	if err := self.ExportCdrsToFile(attr, &efc); err != nil {
+		return err
+	} else if efc.TotalRecords == 0 || len(efc.ExportedFilePath) == 0 {
+		return errors.New("No CDR records to export")
+	}
+	// Create a buffer to write our archive to.
+	buf := new(bytes.Buffer)
+	// Create a new zip archive.
+	w := zip.NewWriter(buf)
+	// read generated file
+	content, err := ioutil.ReadFile(efc.ExportedFilePath)
+	if err != nil {
+		return err
+	}
+	exportFileName := path.Base(efc.ExportedFilePath)
+	f, err := w.Create(exportFileName)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(content)
+	if err != nil {
+		return err
+	}
+	// Write metadata into a separate file with extension .cgr
+	medaData, err := json.MarshalIndent(efc, "", "  ")
+	if err != nil {
+		errors.New("Failed creating metadata content")
+	}
+	medatadaFileName := exportFileName[:len(path.Ext(exportFileName))] + ".cgr"
+	mf, err := w.Create(medatadaFileName)
+	if err != nil {
+		return err
+	}
+	_, err = mf.Write(medaData)
+	if err != nil {
+		return err
+	}
+	// Make sure to check the error on Close.
+	if err := w.Close(); err != nil {
+		return err
+	}
+	if err := os.Remove(efc.ExportedFilePath); err != nil {
+		fmt.Errorf("Failed removing exported file at path: %s", efc.ExportedFilePath)
+	}
+	*reply = base64.StdEncoding.EncodeToString(buf.Bytes())
+	return nil
+}
+
+// Export Cdrs to file
+func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.ExportedFileCdrs) error {
+	var err error
+	exportTemplate := self.Config.CdreProfiles[utils.META_DEFAULT]
+	if attr.ExportTemplate != nil && len(*attr.ExportTemplate) != 0 { // Export template prefered, use it
+		var hasIt bool
+		if exportTemplate, hasIt = self.Config.CdreProfiles[*attr.ExportTemplate]; !hasIt {
+			return fmt.Errorf("%s:ExportTemplate", utils.ErrNotFound.Error())
+		}
+	}
+	if exportTemplate == nil {
+		return fmt.Errorf("%s:ExportTemplate", utils.ErrMandatoryIeMissing.Error())
+	}
+	cdrFormat := exportTemplate.CdrFormat
+	if attr.CdrFormat != nil && len(*attr.CdrFormat) != 0 {
+		cdrFormat = strings.ToLower(*attr.CdrFormat)
+	}
+	if !utils.IsSliceMember(utils.CdreCdrFormats, cdrFormat) {
+		return fmt.Errorf("%s:%s", utils.ErrMandatoryIeMissing.Error(), "CdrFormat")
+	}
+	fieldSep := exportTemplate.FieldSeparator
+	if attr.FieldSeparator != nil && len(*attr.FieldSeparator) != 0 {
+		fieldSep, _ = utf8.DecodeRuneInString(*attr.FieldSeparator)
+		if fieldSep == utf8.RuneError {
+			return fmt.Errorf("%s:FieldSeparator:%s", utils.ErrServerError.Error(), "Invalid")
+		}
+	}
+	exportDir := exportTemplate.ExportDir
+	if attr.ExportDir != nil && len(*attr.ExportDir) != 0 {
+		exportDir = *attr.ExportDir
+	}
+	exportId := strconv.FormatInt(time.Now().Unix(), 10)
+	if attr.ExportId != nil && len(*attr.ExportId) != 0 {
+		exportId = *attr.ExportId
+	}
+	fileName := fmt.Sprintf("cdre_%s.%s", exportId, cdrFormat)
+	if attr.ExportFileName != nil && len(*attr.ExportFileName) != 0 {
+		fileName = *attr.ExportFileName
+	}
+	filePath := path.Join(exportDir, fileName)
+	if cdrFormat == utils.DRYRUN {
+		filePath = utils.DRYRUN
+	}
+	dataUsageMultiplyFactor := exportTemplate.DataUsageMultiplyFactor
+	if attr.DataUsageMultiplyFactor != nil && *attr.DataUsageMultiplyFactor != 0.0 {
+		dataUsageMultiplyFactor = *attr.DataUsageMultiplyFactor
+	}
+	smsUsageMultiplyFactor := exportTemplate.SmsUsageMultiplyFactor
+	if attr.SmsUsageMultiplyFactor != nil && *attr.SmsUsageMultiplyFactor != 0.0 {
+		smsUsageMultiplyFactor = *attr.SmsUsageMultiplyFactor
+	}
+	genericUsageMultiplyFactor := exportTemplate.GenericUsageMultiplyFactor
+	if attr.GenericUsageMultiplyFactor != nil && *attr.GenericUsageMultiplyFactor != 0.0 {
+		genericUsageMultiplyFactor = *attr.GenericUsageMultiplyFactor
+	}
+	costMultiplyFactor := exportTemplate.CostMultiplyFactor
+	if attr.CostMultiplyFactor != nil && *attr.CostMultiplyFactor != 0.0 {
+		costMultiplyFactor = *attr.CostMultiplyFactor
+	}
+	costShiftDigits := exportTemplate.CostShiftDigits
+	if attr.CostShiftDigits != nil {
+		costShiftDigits = *attr.CostShiftDigits
+	}
+	roundingDecimals := exportTemplate.CostRoundingDecimals
+	if attr.RoundDecimals != nil {
+		roundingDecimals = *attr.RoundDecimals
+	}
+	maskDestId := exportTemplate.MaskDestId
+	if attr.MaskDestinationId != nil && len(*attr.MaskDestinationId) != 0 {
+		maskDestId = *attr.MaskDestinationId
+	}
+	maskLen := exportTemplate.MaskLength
+	if attr.MaskLength != nil {
+		maskLen = *attr.MaskLength
+	}
+	cdrsFltr, err := attr.AsCdrsFilter()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	cdrs, _, err := self.CdrDb.GetStoredCdrs(cdrsFltr)
+	if err != nil {
+		return err
+	} else if len(cdrs) == 0 {
+		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
+		return nil
+	}
+	cdrexp, err := cdre.NewCdrExporter(cdrs, self.CdrDb, exportTemplate, cdrFormat, fieldSep, exportId, dataUsageMultiplyFactor, smsUsageMultiplyFactor, genericUsageMultiplyFactor,
+		costMultiplyFactor, costShiftDigits, roundingDecimals, self.Config.RoundingDecimals, maskDestId, maskLen, self.Config.HttpSkipTlsVerify)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if cdrexp.TotalExportedCdrs() == 0 {
+		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
+		return nil
+	}
+	if err := cdrexp.WriteToFile(filePath); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), TotalCost: cdrexp.TotalCost(), FirstOrderId: cdrexp.FirstOrderId(), LastOrderId: cdrexp.LastOrderId()}
+	if !attr.SuppressCgrIds {
+		reply.ExportedCgrIds = cdrexp.PositiveExports()
+		reply.UnexportedCgrIds = cdrexp.NegativeExports()
+	}
+	return nil
+}
+
+// Remove Cdrs out of CDR storage
+func (self *ApierV1) RemCdrs(attrs utils.AttrRemCdrs, reply *string) error {
+	if len(attrs.CgrIds) == 0 {
+		return fmt.Errorf("%s:CgrIds", utils.ErrMandatoryIeMissing.Error())
+	}
+	if err := self.CdrDb.RemStoredCdrs(attrs.CgrIds); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}

apier/v1/cdrs.go

@@ -0,0 +1,60 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Retrieves the callCost out of CGR logDb
+func (apier *ApierV1) GetCallCostLog(attrs utils.AttrGetCallCost, reply *engine.CallCost) error {
+	if attrs.CgrId == "" {
+		return utils.NewErrMandatoryIeMissing("CgrId")
+	}
+	if attrs.RunId == "" {
+		attrs.RunId = utils.META_DEFAULT
+	}
+	if cc, err := apier.CdrDb.GetCallCostLog(attrs.CgrId, "", attrs.RunId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if cc == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = *cc
+	}
+	return nil
+}
+
+// Retrieves CDRs based on the filters
+func (apier *ApierV1) GetCdrs(attrs utils.AttrGetCdrs, reply *[]*engine.ExternalCdr) error {
+	cdrsFltr, err := attrs.AsCdrsFilter()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if cdrs, _, err := apier.CdrDb.GetStoredCdrs(cdrsFltr); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(cdrs) == 0 {
+		*reply = make([]*engine.ExternalCdr, 0)
+	} else {
+		for _, cdr := range cdrs {
+			*reply = append(*reply, cdr.AsExternalCdr())
+		}
+	}
+	return nil
+}

apier/v1/cdrstatsv1.go

@@ -16,7 +16,7 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
 	"fmt"
@@ -27,7 +27,7 @@ import (
 
 // Interact with Stats server
 type CDRStatsV1 struct {
-	CdrStats *engine.Stats
+	CdrStats engine.StatsInterface
 }
 
 type AttrGetMetrics struct {
@@ -36,7 +36,7 @@ type AttrGetMetrics struct {
 
 func (sts *CDRStatsV1) GetMetrics(attr AttrGetMetrics, reply *map[string]float64) error {
 	if len(attr.StatsQueueId) == 0 {
-		return fmt.Errorf("%s:StatsQueueId", utils.ERR_MANDATORY_IE_MISSING)
+		return fmt.Errorf("%s:StatsQueueId", utils.ErrMandatoryIeMissing.Error())
 	}
 	return sts.CdrStats.GetValues(attr.StatsQueueId, reply)
 }
@@ -45,6 +45,14 @@ func (sts *CDRStatsV1) GetQueueIds(empty string, reply *[]string) error {
 	return sts.CdrStats.GetQueueIds(0, reply)
 }
 
+func (sts *CDRStatsV1) GetQueue(id string, sq *engine.StatsQueue) error {
+	return sts.CdrStats.GetQueue(id, sq)
+}
+
+func (sts *CDRStatsV1) GetQueueTriggers(id string, ats *engine.ActionTriggerPriotityList) error {
+	return sts.CdrStats.GetQueueTriggers(id, ats)
+}
+
 func (sts *CDRStatsV1) ReloadQueues(attr utils.AttrCDRStatsReloadQueues, reply *string) error {
 	if err := sts.CdrStats.ReloadQueues(attr.StatsQueueIds, nil); err != nil {
 		return err

apier/v1/cdrstatsv1_local_test.go

@@ -16,50 +16,44 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
 	"fmt"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
 	"net/http"
 	"net/rpc"
 	"net/rpc/jsonrpc"
-	"os/exec"
 	"path"
 	"reflect"
 	"testing"
 	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
 )
 
 var cdrstCfgPath string
 var cdrstCfg *config.CGRConfig
 var cdrstRpc *rpc.Client
 
-func init() {
-	cdrstCfgPath = path.Join(*dataDir, "conf", "samples", "cdrstatsv1_local_test.cfg")
-	cdrstCfg, _ = config.NewCGRConfigFromFile(&cdrstCfgPath)
+func TestCDRStatsLclLoadConfig(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrstCfgPath = path.Join(*dataDir, "conf", "samples", "cdrstats")
+	if cdrstCfg, err = config.NewCGRConfigFromFolder(cfgPath); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestCDRStatsLclInitDataDb(t *testing.T) {
 	if !*testLocal {
 		return
 	}
-	ratingDb, err := engine.ConfigureRatingStorage(cdrstCfg.RatingDBType, cdrstCfg.RatingDBHost, cdrstCfg.RatingDBPort, cdrstCfg.RatingDBName,
-		cdrstCfg.RatingDBUser, cdrstCfg.RatingDBPass, cdrstCfg.DBDataEncoding)
-	if err != nil {
-		t.Fatal("Cannot connect to dataDb", err)
-	}
-	accountDb, err := engine.ConfigureAccountingStorage(cdrstCfg.AccountDBType, cdrstCfg.AccountDBHost, cdrstCfg.AccountDBPort, cdrstCfg.AccountDBName,
-		cdrstCfg.AccountDBUser, cdrstCfg.AccountDBPass, cdrstCfg.DBDataEncoding)
-	if err != nil {
-		t.Fatal("Cannot connect to dataDb", err)
-	}
-	for _, db := range []engine.Storage{ratingDb, accountDb} {
-		if err := db.Flush(); err != nil {
-			t.Fatal("Cannot reset dataDb", err)
-		}
+	if err := engine.InitDataDb(cdrstCfg); err != nil {
+		t.Fatal(err)
 	}
 }
 
@@ -67,16 +61,9 @@ func TestCDRStatsLclStartEngine(t *testing.T) {
 	if !*testLocal {
 		return
 	}
-	enginePath, err := exec.LookPath("cgr-engine")
-	if err != nil {
-		t.Fatal("Cannot find cgr-engine executable")
+	if _, err := engine.StopStartEngine(cdrstCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
 	}
-	exec.Command("pkill", "cgr-engine").Run() // Just to make sure another one is not running, bit brutal maybe we can fine tune it
-	engine := exec.Command(enginePath, "-config", cdrstCfgPath)
-	if err := engine.Start(); err != nil {
-		t.Fatal("Cannot start cgr-engine: ", err.Error())
-	}
-	time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time to rater to fire up
 }
 
 // Connect rpc client to rater
@@ -91,19 +78,6 @@ func TestCDRStatsLclRpcConn(t *testing.T) {
 	}
 }
 
-func TestCDRStatsLclGetQueueIds(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	var queueIds []string
-	eQueueIds := []string{"*default"}
-	if err := cdrstRpc.Call("CDRStatsV1.GetQueueIds", "", &queueIds); err != nil {
-		t.Error("Calling CDRStatsV1.GetQueueIds, got error: ", err.Error())
-	} else if !reflect.DeepEqual(eQueueIds, queueIds) {
-		t.Errorf("Expecting: %v, received: %v", eQueueIds, queueIds)
-	}
-}
-
 func TestCDRStatsLclLoadTariffPlanFromFolder(t *testing.T) {
 	if !*testLocal {
 		return
@@ -124,10 +98,10 @@ func TestCDRStatsLclGetQueueIds2(t *testing.T) {
 		return
 	}
 	var queueIds []string
-	eQueueIds := []string{"*default", "CDRST3", "CDRST4"}
+	eQueueIds := []string{"CDRST3", "CDRST4"}
 	if err := cdrstRpc.Call("CDRStatsV1.GetQueueIds", "", &queueIds); err != nil {
 		t.Error("Calling CDRStatsV1.GetQueueIds, got error: ", err.Error())
-	} else if !reflect.DeepEqual(eQueueIds, queueIds) {
+	} else if len(eQueueIds) != len(queueIds) {
 		t.Errorf("Expecting: %v, received: %v", eQueueIds, queueIds)
 	}
 }
@@ -137,35 +111,35 @@ func TestCDRStatsLclPostCdrs(t *testing.T) {
 		return
 	}
 	httpClient := new(http.Client)
-	storedCdrs := []*utils.StoredCdr{
-		&utils.StoredCdr{CgrId: utils.Sha1("dsafdsafa", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
-			ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+	storedCdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsafa", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 			Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(),
 			AnswerTime: time.Now(), MediationRunId: utils.DEFAULT_RUNID,
 			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
 			Cost: 1.01, RatedAccount: "dan", RatedSubject: "dan",
 		},
-		&utils.StoredCdr{CgrId: utils.Sha1("dsafdsafb", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
-			ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsafb", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 			Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(),
 			AnswerTime: time.Now(), MediationRunId: utils.DEFAULT_RUNID,
 			Usage: time.Duration(5) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dan",
 		},
-		&utils.StoredCdr{CgrId: utils.Sha1("dsafdsafc", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
-			ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsafc", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 			Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), AnswerTime: time.Now(),
 			MediationRunId: utils.DEFAULT_RUNID,
 			Usage:          time.Duration(30) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dan",
 		},
-		&utils.StoredCdr{CgrId: utils.Sha1("dsafdsafd", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
-			ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsafd", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 			Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), AnswerTime: time.Time{},
 			MediationRunId: utils.DEFAULT_RUNID,
 			Usage:          time.Duration(0) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dan",
 		},
 	}
 	for _, storedCdr := range storedCdrs {
-		if _, err := httpClient.PostForm(fmt.Sprintf("http://%s/cgr", "127.0.0.1:2080"), storedCdr.AsHttpForm()); err != nil {
+		if _, err := httpClient.PostForm(fmt.Sprintf("http://%s/cdr_http", "127.0.0.1:2080"), storedCdr.AsHttpForm()); err != nil {
 			t.Error(err.Error())
 		}
 	}
@@ -177,13 +151,6 @@ func TestCDRStatsLclGetMetrics1(t *testing.T) {
 	if !*testLocal {
 		return
 	}
-	var rcvMetrics1 map[string]float64
-	expectedMetrics1 := map[string]float64{"ASR": 75, "ACD": 15, "ACC": 15}
-	if err := cdrstRpc.Call("CDRStatsV1.GetMetrics", AttrGetMetrics{StatsQueueId: "*default"}, &rcvMetrics1); err != nil {
-		t.Error("Calling CDRStatsV1.GetMetrics, got error: ", err.Error())
-	} else if !reflect.DeepEqual(expectedMetrics1, rcvMetrics1) {
-		t.Errorf("Expecting: %v, received: %v", expectedMetrics1, rcvMetrics1)
-	}
 	var rcvMetrics2 map[string]float64
 	expectedMetrics2 := map[string]float64{"ASR": 75, "ACD": 15}
 	if err := cdrstRpc.Call("CDRStatsV1.GetMetrics", AttrGetMetrics{StatsQueueId: "CDRST4"}, &rcvMetrics2); err != nil {
@@ -193,6 +160,29 @@ func TestCDRStatsLclGetMetrics1(t *testing.T) {
 	}
 }
 
+// Test stats persistence
+func TestCDRStatsLclStatsPersistence(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	time.Sleep(time.Duration(2) * time.Second) // Allow stats to be updated in dataDb
+	if _, err := engine.StopStartEngine(cdrstCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+	var err error
+	cdrstRpc, err = jsonrpc.Dial("tcp", cdrstCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+	var rcvMetrics map[string]float64
+	expectedMetrics := map[string]float64{"ASR": 75, "ACD": 15}
+	if err := cdrstRpc.Call("CDRStatsV1.GetMetrics", AttrGetMetrics{StatsQueueId: "CDRST4"}, &rcvMetrics); err != nil {
+		t.Error("Calling CDRStatsV1.GetMetrics, got error: ", err.Error())
+	} else if !reflect.DeepEqual(expectedMetrics, rcvMetrics) {
+		t.Errorf("Expecting: %v, received: %v", expectedMetrics, rcvMetrics)
+	}
+}
+
 func TestCDRStatsLclResetMetrics(t *testing.T) {
 	if !*testLocal {
 		return
@@ -204,15 +194,8 @@ func TestCDRStatsLclResetMetrics(t *testing.T) {
 		t.Error("Unexpected reply received: ", reply)
 	}
 	time.Sleep(time.Duration(*waitRater) * time.Millisecond)
-	var rcvMetrics1 map[string]float64
-	expectedMetrics1 := map[string]float64{"ASR": 75, "ACD": 15, "ACC": 15}
-	if err := cdrstRpc.Call("CDRStatsV1.GetMetrics", AttrGetMetrics{StatsQueueId: "*default"}, &rcvMetrics1); err != nil {
-		t.Error("Calling CDRStatsV1.GetMetrics, got error: ", err.Error())
-	} else if !reflect.DeepEqual(expectedMetrics1, rcvMetrics1) {
-		t.Errorf("Expecting: %v, received: %v", expectedMetrics1, rcvMetrics1)
-	}
 	var rcvMetrics2 map[string]float64
-	expectedMetrics2 := map[string]float64{"ASR": 0, "ACD": 0}
+	expectedMetrics2 := map[string]float64{"ASR": -1, "ACD": -1}
 	if err := cdrstRpc.Call("CDRStatsV1.GetMetrics", AttrGetMetrics{StatsQueueId: "CDRST4"}, &rcvMetrics2); err != nil {
 		t.Error("Calling CDRStatsV1.GetMetrics, got error: ", err.Error())
 	} else if !reflect.DeepEqual(expectedMetrics2, rcvMetrics2) {

apier/v1/cdrsv1.go

@@ -1,14 +1,14 @@
 /*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
 
-This program is free software: you can redistribute it and/or modify
+This program is free software: you can Storagetribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.
 
 This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
+but WITH*out ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
 
@@ -16,25 +16,40 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"fmt"
 	"time"
 
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
-type MediatorV1 struct {
-	Medi *engine.Mediator
+// Receive CDRs via RPC methods
+type CdrsV1 struct {
+	CdrSrv *engine.CdrServer
+}
+
+// Designed for CGR internal usage
+func (self *CdrsV1) ProcessCdr(cdr *engine.StoredCdr, reply *string) error {
+	if err := self.CdrSrv.ProcessCdr(cdr); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
+}
+
+// Designed for external programs feeding CDRs to CGRateS
+func (self *CdrsV1) ProcessExternalCdr(cdr *engine.ExternalCdr, reply *string) error {
+	if err := self.CdrSrv.ProcessExternalCdr(cdr); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
 }
 
 // Remotely start mediation with specific runid, runs asynchronously, it's status will be displayed in syslog
-func (self *MediatorV1) RateCdrs(attrs utils.AttrRateCdrs, reply *string) error {
-	if self.Medi == nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, "MediatorNotRunning")
-	}
+func (self *CdrsV1) RateCdrs(attrs utils.AttrRateCdrs, reply *string) error {
 	var tStart, tEnd time.Time
 	var err error
 	if len(attrs.TimeStart) != 0 {
@@ -49,10 +64,10 @@ func (self *MediatorV1) RateCdrs(attrs utils.AttrRateCdrs, reply *string) error
 	}
 	//RateCdrs(cgrIds, runIds, tors, cdrHosts, cdrSources, reqTypes, directions, tenants, categories, accounts, subjects, destPrefixes []string,
 	//orderIdStart, orderIdEnd int64, timeStart, timeEnd time.Time, rerateErrors, rerateRated bool)
-	if err := self.Medi.RateCdrs(attrs.CgrIds, attrs.MediationRunIds, attrs.TORs, attrs.CdrHosts, attrs.CdrSources, attrs.ReqTypes, attrs.Directions,
+	if err := self.CdrSrv.RateCdrs(attrs.CgrIds, attrs.MediationRunIds, attrs.TORs, attrs.CdrHosts, attrs.CdrSources, attrs.ReqTypes, attrs.Directions,
 		attrs.Tenants, attrs.Categories, attrs.Accounts, attrs.Subjects, attrs.DestinationPrefixes, attrs.RatedAccounts, attrs.RatedSubjects,
 		attrs.OrderIdStart, attrs.OrderIdEnd, tStart, tEnd, attrs.RerateErrors, attrs.RerateRated, attrs.SendToStats); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	}
 	*reply = utils.OK
 	return nil

apier/v1/costs.go

@@ -1,6 +1,6 @@
 /*
 Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,13 +16,13 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"fmt"
+	"time"
+
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
-	"time"
 )
 
 type AttrGetDataCost struct {
@@ -35,7 +35,7 @@ type AttrGetDataCost struct {
 
 func (apier *ApierV1) GetDataCost(attrs AttrGetDataCost, reply *engine.DataCost) error {
 	usageAsDuration := time.Duration(attrs.Usage) * time.Second // Convert to seconds to match the loaded rates
-	cd := engine.CallDescriptor{
+	cd := &engine.CallDescriptor{
 		Direction:     attrs.Direction,
 		Category:      attrs.Category,
 		Tenant:        attrs.Tenant,
@@ -48,10 +48,10 @@ func (apier *ApierV1) GetDataCost(attrs AttrGetDataCost, reply *engine.DataCost)
 	}
 	var cc engine.CallCost
 	if err := apier.Responder.GetCost(cd, &cc); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	}
 	if dc, err := cc.ToDataCost(); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	} else if dc != nil {
 		*reply = *dc
 	}

apier/v1/debit.go

@@ -0,0 +1,67 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func (self *ApierV1) DebitUsage(usageRecord engine.UsageRecord, reply *string) error {
+	if missing := utils.MissingStructFields(&usageRecord, []string{"Account", "Destination", "Usage"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	out, err := engine.LoadUserProfile(usageRecord, "")
+	if err != nil {
+		*reply = err.Error()
+		return err
+	}
+	usageRecord = out.(engine.UsageRecord)
+	if usageRecord.TOR == "" {
+		usageRecord.TOR = utils.VOICE
+	}
+	if usageRecord.ReqType == "" {
+		usageRecord.ReqType = self.Config.DefaultReqType
+	}
+	if usageRecord.Direction == "" {
+		usageRecord.Direction = utils.OUT
+	}
+	if usageRecord.Tenant == "" {
+		usageRecord.Tenant = self.Config.DefaultTenant
+	}
+	if usageRecord.Category == "" {
+		usageRecord.Category = self.Config.DefaultCategory
+	}
+	if usageRecord.Subject == "" {
+		usageRecord.Subject = usageRecord.Account
+	}
+	if usageRecord.AnswerTime == "" {
+		usageRecord.AnswerTime = utils.META_NOW
+	}
+	cd, err := usageRecord.AsCallDescriptor()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	var cc engine.CallCost
+	if err := self.Responder.Debit(cd, &cc); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = OK
+	return nil
+}

apier/v1/derivedcharging.go

@@ -1,6 +1,6 @@
 /*
 Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can Storagetribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,10 +16,11 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
 	"fmt"
+
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
@@ -27,10 +28,10 @@ import (
 // Get DerivedChargers applying to our call, appends general configured to account specific ones if that is configured
 func (self *ApierV1) GetDerivedChargers(attrs utils.AttrDerivedChargers, reply *utils.DerivedChargers) (err error) {
 	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Direction", "Account", "Subject"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if hDc, err := engine.HandleGetDerivedChargers(self.AccountDb, self.Config, attrs); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if hDc, err := engine.HandleGetDerivedChargers(self.RatingDb, &attrs); err != nil {
+		return utils.NewErrServerError(err)
 	} else if hDc != nil {
 		*reply = hDc
 	}
@@ -40,23 +41,45 @@ func (self *ApierV1) GetDerivedChargers(attrs utils.AttrDerivedChargers, reply *
 type AttrSetDerivedChargers struct {
 	Direction, Tenant, Category, Account, Subject string
 	DerivedChargers                               utils.DerivedChargers
+	Overwrite                                     bool // Do not overwrite if present in redis
 }
 
 func (self *ApierV1) SetDerivedChargers(attrs AttrSetDerivedChargers, reply *string) (err error) {
-	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Category", "Direction", "Account", "Subject", "DerivedChargers"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+	if len(attrs.Direction) == 0 {
+		attrs.Direction = utils.OUT
+	}
+	if len(attrs.Tenant) == 0 {
+		attrs.Tenant = utils.ANY
+	}
+	if len(attrs.Category) == 0 {
+		attrs.Category = utils.ANY
+	}
+	if len(attrs.Account) == 0 {
+		attrs.Account = utils.ANY
+	}
+	if len(attrs.Subject) == 0 {
+		attrs.Subject = utils.ANY
 	}
 	for _, dc := range attrs.DerivedChargers {
 		if _, err = utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP); err != nil { // Make sure rules are OK before loading in db
-			return fmt.Errorf("%s:%s", utils.ERR_PARSER_ERROR, err.Error())
+			return fmt.Errorf("%s:%s", utils.ErrParserError.Error(), err.Error())
 		}
 	}
 	dcKey := utils.DerivedChargersKey(attrs.Direction, attrs.Tenant, attrs.Category, attrs.Account, attrs.Subject)
-	if err := self.AccountDb.SetDerivedChargers(dcKey, attrs.DerivedChargers); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if !attrs.Overwrite {
+		if exists, err := self.RatingDb.HasData(utils.DESTINATION_PREFIX, dcKey); err != nil {
+			return utils.NewErrServerError(err)
+		} else if exists {
+			return utils.ErrExists
+		}
 	}
-	if err := self.AccountDb.CacheAccounting([]string{}, []string{}, []string{}, []string{engine.DERIVEDCHARGERS_PREFIX + dcKey}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.RatingDb.SetDerivedChargers(dcKey, attrs.DerivedChargers); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if err := self.RatingDb.CachePrefixValues(map[string][]string{
+		utils.DERIVEDCHARGERS_PREFIX: []string{utils.DERIVEDCHARGERS_PREFIX + dcKey},
+	}); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = utils.OK
 	return nil
@@ -67,16 +90,28 @@ type AttrRemDerivedChargers struct {
 }
 
 func (self *ApierV1) RemDerivedChargers(attrs AttrRemDerivedChargers, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"Direction", "Tenant", "Category", "Account", "Subject"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+	if len(attrs.Direction) == 0 {
+		attrs.Direction = utils.OUT
 	}
-	if err := self.AccountDb.SetDerivedChargers(utils.DerivedChargersKey(attrs.Direction, attrs.Tenant, attrs.Category, attrs.Account, attrs.Subject), nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if len(attrs.Tenant) == 0 {
+		attrs.Tenant = utils.ANY
+	}
+	if len(attrs.Category) == 0 {
+		attrs.Category = utils.ANY
+	}
+	if len(attrs.Account) == 0 {
+		attrs.Account = utils.ANY
+	}
+	if len(attrs.Subject) == 0 {
+		attrs.Subject = utils.ANY
+	}
+	if err := self.RatingDb.SetDerivedChargers(utils.DerivedChargersKey(attrs.Direction, attrs.Tenant, attrs.Category, attrs.Account, attrs.Subject), nil); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}
-	if err := self.AccountDb.CacheAccounting([]string{}, []string{}, []string{}, nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.RatingDb.CachePrefixes(utils.DERIVEDCHARGERS_PREFIX); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	return nil
 }

apier/v1/derivedcharging_test.go

@@ -1,6 +1,6 @@
 /*
 Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
+Copyright (C) ITsysCOM GmbH
 
 This program is free software: you can Storagetribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,15 +16,15 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"fmt"
+	"reflect"
+	"testing"
+
 	"github.com/cgrates/cgrates/config"
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
-	"reflect"
-	"testing"
 )
 
 var apierDcT *ApierV1
@@ -32,9 +32,10 @@ var apierDcT *ApierV1
 func init() {
 	dataStorage, _ := engine.NewMapStorage()
 	cfg, _ := config.NewDefaultCGRConfig()
-	apierDcT = &ApierV1{AccountDb: engine.AccountingStorage(dataStorage), Config: cfg}
+	apierDcT = &ApierV1{RatingDb: engine.RatingStorage(dataStorage), Config: cfg}
 }
 
+/*
 func TestGetEmptyDC(t *testing.T) {
 	attrs := utils.AttrDerivedChargers{Tenant: "cgrates.org", Category: "call", Direction: "*out", Account: "dan", Subject: "dan"}
 	var dcs utils.DerivedChargers
@@ -44,6 +45,7 @@ func TestGetEmptyDC(t *testing.T) {
 		t.Error("Returned DerivedChargers not matching the configured ones")
 	}
 }
+*/
 
 func TestSetDC(t *testing.T) {
 	dcs1 := utils.DerivedChargers{
@@ -87,6 +89,7 @@ func TestRemDC(t *testing.T) {
 	}
 }
 
+/*
 func TestGetEmptyDC2(t *testing.T) {
 	attrs := utils.AttrDerivedChargers{Tenant: "cgrates.org", Category: "call", Direction: "*out", Account: "dan", Subject: "dan"}
 	var dcs utils.DerivedChargers
@@ -99,3 +102,4 @@ func TestGetEmptyDC2(t *testing.T) {
 		t.Error("Returned DerivedChargers not matching the configured ones")
 	}
 }
+*/

apier/v1/lcr.go

@@ -0,0 +1,78 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"fmt"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Computes the LCR for a specific request emulating a call
+func (self *ApierV1) GetLcr(lcrReq engine.LcrRequest, lcrReply *engine.LcrReply) error {
+	cd, err := lcrReq.AsCallDescriptor()
+	if err != nil {
+		return err
+	}
+	var lcrQried engine.LCRCost
+	if err := self.Responder.GetLCR(cd, &lcrQried); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if lcrQried.Entry == nil {
+		return utils.ErrNotFound
+	}
+	if lcrQried.HasErrors() {
+		lcrQried.LogErrors()
+		return fmt.Errorf("%s:%s", utils.ErrServerError.Error(), "LCR_COMPUTE_ERRORS")
+	}
+	lcrReply.DestinationId = lcrQried.Entry.DestinationId
+	lcrReply.RPCategory = lcrQried.Entry.RPCategory
+	lcrReply.Strategy = lcrQried.Entry.Strategy
+	for _, qriedSuppl := range lcrQried.SupplierCosts {
+		if dtcs, err := utils.NewDTCSFromRPKey(qriedSuppl.Supplier); err != nil {
+			return utils.NewErrServerError(err)
+		} else {
+			lcrReply.Suppliers = append(lcrReply.Suppliers, &engine.LcrSupplier{Supplier: dtcs.Subject, Cost: qriedSuppl.Cost, QOS: qriedSuppl.QOS})
+		}
+	}
+	return nil
+}
+
+// Computes the LCR for a specific request emulating a call, returns a comma separated list of suppliers
+func (self *ApierV1) GetLcrSuppliers(lcrReq engine.LcrRequest, suppliers *string) (err error) {
+	cd, err := lcrReq.AsCallDescriptor()
+	if err != nil {
+		return err
+	}
+	var lcrQried engine.LCRCost
+	if err := self.Responder.GetLCR(cd, &lcrQried); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if lcrQried.HasErrors() {
+		lcrQried.LogErrors()
+		return fmt.Errorf("%s:%s", utils.ErrServerError.Error(), "LCR_ERRORS")
+	}
+	if suppliersStr, err := lcrQried.SuppliersString(); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*suppliers = suppliersStr
+	}
+	return nil
+}

apier/v1/mediator.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,27 +16,4 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
-
-// Tariff plan related APIs
-
-import (
-	"errors"
-	"fmt"
-	"github.com/cgrates/cgrates/utils"
-)
-
-type AttrGetTPIds struct {
-}
-
-// Queries tarrif plan identities gathered from all tables.
-func (self *ApierV1) GetTPIds(attrs AttrGetTPIds, reply *[]string) error {
-	if ids, err := self.StorDb.GetTPIds(); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
-	} else {
-		*reply = ids
-	}
-	return nil
-}
+package v1

apier/v1/scheduler.go

@@ -1,6 +1,6 @@
 /*
 Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,12 +16,14 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
 	"errors"
-	"github.com/cgrates/cgrates/utils"
+	"strings"
 	"time"
+
+	"github.com/cgrates/cgrates/utils"
 )
 
 /*
@@ -98,6 +100,7 @@ import (
 type AttrsGetScheduledActions struct {
 	Direction, Tenant, Account string
 	TimeStart, TimeEnd         time.Time // Filter based on next runTime
+	utils.Paginator
 }
 
 type ScheduledActions struct {
@@ -111,8 +114,25 @@ func (self *ApierV1) GetScheduledActions(attrs AttrsGetScheduledActions, reply *
 	if self.Sched == nil {
 		return errors.New("SCHEDULER_NOT_ENABLED")
 	}
-	for _, qActions := range self.Sched.GetQueue() {
+	scheduledActions := self.Sched.GetQueue()
+	var min, max int
+	if attrs.Paginator.Offset != nil {
+		min = *attrs.Paginator.Offset
+	}
+	if attrs.Paginator.Limit != nil {
+		max = *attrs.Paginator.Limit
+	}
+	if max > len(scheduledActions) {
+		max = len(scheduledActions)
+	}
+	scheduledActions = scheduledActions[min : min+max]
+	for _, qActions := range scheduledActions {
 		sas := &ScheduledActions{ActionsId: qActions.ActionsId, ActionPlanId: qActions.Id, ActionPlanUuid: qActions.Uuid}
+		if attrs.SearchTerm != "" &&
+			!(strings.Contains(sas.ActionPlanId, attrs.SearchTerm) ||
+				strings.Contains(sas.ActionsId, attrs.SearchTerm)) {
+			continue
+		}
 		sas.NextRunTime = qActions.GetNextStartTime(time.Now())
 		if !attrs.TimeStart.IsZero() && sas.NextRunTime.Before(attrs.TimeStart) {
 			continue // Filter here only requests in the filtered interval

apier/v1/smv1.go

@@ -0,0 +1,47 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/sessionmanager"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Interact with SessionManager
+type SessionManagerV1 struct {
+	SMs []sessionmanager.SessionManager // List of session managers since we support having more than one active session manager running on one host
+}
+
+func (self *SessionManagerV1) ActiveSessionMangers(ignored string, reply *[]sessionmanager.SessionManager) error {
+	if len(self.SMs) == 0 {
+		return utils.ErrNotFound
+	}
+	*reply = self.SMs
+	return nil
+}
+
+func (self *SessionManagerV1) ActiveSessions(attrs utils.AttrGetSMASessions, reply *[]*sessionmanager.ActiveSession) error {
+	if attrs.SessionManagerIndex > len(self.SMs)-1 {
+		return utils.ErrNotFound
+	}
+	for _, session := range self.SMs[attrs.SessionManagerIndex].Sessions() {
+		*reply = append(*reply, session.AsActiveSessions()...)
+	}
+	return nil
+}

apier/v1/tp.go

@@ -0,0 +1,99 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+// Tariff plan related APIs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type AttrGetTPIds struct {
+}
+
+// Queries tarrif plan identities gathered from all tables.
+func (self *ApierV1) GetTPIds(attrs AttrGetTPIds, reply *[]string) error {
+	if ids, err := self.StorDb.GetTpIds(); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+type AttrImportTPZipFile struct {
+	TPid string
+	File []byte
+}
+
+func (self *ApierV1) ImportTPZipFile(attrs AttrImportTPZipFile, reply *string) error {
+	tmpDir, err := ioutil.TempDir("/tmp", "cgr_")
+	if err != nil {
+		*reply = "ERROR: creating temp directory!"
+		return err
+	}
+	zipFile := filepath.Join(tmpDir, "/file.zip")
+	if err = ioutil.WriteFile(zipFile, attrs.File, os.ModePerm); err != nil {
+		*reply = "ERROR: writing zip file!"
+		return err
+	}
+	if err = utils.Unzip(zipFile, tmpDir); err != nil {
+		*reply = "ERROR: unziping file!"
+		return err
+	}
+	csvfilesFound := false
+	if err = filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error {
+		if !info.IsDir() {
+			return nil
+		}
+		csvFiles, err := filepath.Glob(filepath.Join(path, "*csv"))
+		if csvFiles != nil {
+			if attrs.TPid == "" {
+				*reply = "ERROR: missing TPid!"
+				return err
+			}
+			csvImporter := engine.TPCSVImporter{
+				TPid:     attrs.TPid,
+				StorDb:   self.StorDb,
+				DirPath:  path,
+				Sep:      ',',
+				Verbose:  false,
+				ImportId: "",
+			}
+			if errImport := csvImporter.Run(); errImport != nil {
+				return errImport
+			}
+			csvfilesFound = true
+		}
+		return err
+	}); err != nil || !csvfilesFound {
+		*reply = "ERROR: finding csv files!"
+		return err
+	}
+	os.RemoveAll(tmpDir)
+	*reply = "OK"
+	return nil
+}

apier/v1/tpaccountactions.go

@@ -0,0 +1,166 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new AccountActions profile within a tariff plan
+func (self *ApierV1) SetTPAccountActions(attrs utils.TPAccountActions, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs,
+		[]string{"TPid", "LoadId", "Tenant", "Account", "Direction", "ActionPlanId", "ActionTriggersId"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	aas := engine.APItoModelAccountAction(&attrs)
+	if err := self.StorDb.SetTpAccountActions([]engine.TpAccountAction{*aas}); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPAccountActionsByLoadId struct {
+	TPid   string // Tariff plan id
+	LoadId string // AccountActions id
+}
+
+// Queries specific AccountActions profile on tariff plan
+func (self *ApierV1) GetTPAccountActionsByLoadId(attrs utils.TPAccountActions, reply *[]*utils.TPAccountActions) error {
+	mndtryFlds := []string{"TPid", "LoadId"}
+	if len(attrs.Account) != 0 { // If account provided as filter, make all related fields mandatory
+		mndtryFlds = append(mndtryFlds, "Tenant", "Account", "Direction")
+	}
+	if missing := utils.MissingStructFields(&attrs, mndtryFlds); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	aas := engine.APItoModelAccountAction(&attrs)
+	if aa, err := self.StorDb.GetTpAccountActions(aas); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(aa) == 0 {
+		return utils.ErrNotFound
+	} else {
+
+		tpAa, err := engine.TpAccountActions(aa).GetAccountActions()
+		if err != nil {
+			return err
+		}
+		var acts []*utils.TPAccountActions
+		if len(attrs.Account) != 0 {
+			acts = []*utils.TPAccountActions{tpAa[attrs.KeyId()]}
+		} else {
+			for _, actLst := range tpAa {
+				acts = append(acts, actLst)
+			}
+		}
+		*reply = acts
+	}
+	return nil
+}
+
+type AttrGetTPAccountActions struct {
+	TPid             string // Tariff plan id
+	AccountActionsId string // DerivedCharge id
+}
+
+// Queries specific DerivedCharge on tariff plan
+func (self *ApierV1) GetTPAccountActions(attrs AttrGetTPAccountActions, reply *utils.TPAccountActions) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "AccountActionsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpAa := &utils.TPAccountActions{TPid: attrs.TPid}
+	if err := tmpAa.SetAccountActionsId(attrs.AccountActionsId); err != nil {
+		return err
+	}
+	tmpAaa := engine.APItoModelAccountAction(tmpAa)
+	if aas, err := self.StorDb.GetTpAccountActions(tmpAaa); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(aas) == 0 {
+		return utils.ErrNotFound
+	} else {
+		tpAaa, err := engine.TpAccountActions(aas).GetAccountActions()
+		if err != nil {
+			return err
+		}
+		aa := tpAaa[tmpAa.KeyId()]
+		tpdc := utils.TPAccountActions{
+			TPid:             attrs.TPid,
+			ActionPlanId:     aa.ActionPlanId,
+			ActionTriggersId: aa.ActionTriggersId,
+		}
+		if err := tpdc.SetAccountActionsId(attrs.AccountActionsId); err != nil {
+			return err
+		}
+		*reply = tpdc
+	}
+	return nil
+}
+
+type AttrGetTPAccountActionIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries AccountActions identities on specific tariff plan.
+func (self *ApierV1) GetTPAccountActionLoadIds(attrs AttrGetTPAccountActionIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACCOUNT_ACTIONS, utils.TPDistinctIds{"loadid"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Queries DerivedCharges identities on specific tariff plan.
+func (self *ApierV1) GetTPAccountActionIds(attrs AttrGetTPAccountActionIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACCOUNT_ACTIONS, utils.TPDistinctIds{"loadid", "direction", "tenant", "account"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific AccountActions on Tariff plan
+func (self *ApierV1) RemTPAccountActions(attrs AttrGetTPAccountActions, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "Account", "Direction"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	aa := engine.TpAccountAction{Tpid: attrs.TPid}
+	if err := aa.SetAccountActionId(attrs.AccountActionsId); err != nil {
+		return err
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.Tpid, aa.Loadid, aa.Direction, aa.Tenant, aa.Account); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpactionplans.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,28 +16,29 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"errors"
 	"fmt"
 
+	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
 // Creates a new ActionTimings profile within a tariff plan
 func (self *ApierV1) SetTPActionPlan(attrs utils.TPActionPlan, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Id", "ActionPlan"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionPlanId", "ActionPlan"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	for _, at := range attrs.ActionPlan {
 		requiredFields := []string{"ActionsId", "TimingId", "Weight"}
 		if missing := utils.MissingStructFields(at, requiredFields); len(missing) != 0 {
-			return fmt.Errorf("%s:Action:%s:%v", utils.ERR_MANDATORY_IE_MISSING, at.ActionsId, missing)
+			return fmt.Errorf("%s:Action:%s:%v", utils.ErrMandatoryIeMissing.Error(), at.ActionsId, missing)
 		}
 	}
-	if err := self.StorDb.SetTPActionTimings(attrs.TPid, map[string][]*utils.TPActionTiming{attrs.Id: attrs.ActionPlan}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	ap := engine.APItoModelActionPlan(&attrs)
+	if err := self.StorDb.SetTpActionPlans(ap); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -45,20 +46,28 @@ func (self *ApierV1) SetTPActionPlan(attrs utils.TPActionPlan, reply *string) er
 
 type AttrGetTPActionPlan struct {
 	TPid string // Tariff plan id
-	Id   string // ActionTimings id
+	Id   string // ActionPlans id
 }
 
 // Queries specific ActionPlan profile on tariff plan
 func (self *ApierV1) GetTPActionPlan(attrs AttrGetTPActionPlan, reply *utils.TPActionPlan) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Id"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ats, err := self.StorDb.GetTPActionTimings(attrs.TPid, attrs.Id); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ats, err := self.StorDb.GetTpActionPlans(attrs.TPid, attrs.Id); err != nil {
+		return utils.NewErrServerError(err)
 	} else if len(ats) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else { // Got the data we need, convert it
-		atRply := &utils.TPActionPlan{attrs.TPid, attrs.Id, ats[attrs.Id]}
+		aps, err := engine.TpActionPlans(ats).GetActionPlans()
+		if err != nil {
+			return err
+		}
+		atRply := &utils.TPActionPlan{
+			TPid:         attrs.TPid,
+			ActionPlanId: attrs.Id,
+			ActionPlan:   aps[attrs.Id],
+		}
 		*reply = *atRply
 	}
 	return nil
@@ -66,17 +75,18 @@ func (self *ApierV1) GetTPActionPlan(attrs AttrGetTPActionPlan, reply *utils.TPA
 
 type AttrGetTPActionPlanIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries ActionPlan identities on specific tariff plan.
 func (self *ApierV1) GetTPActionPlanIds(attrs AttrGetTPActionPlanIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_ACTION_PLANS, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACTION_PLANS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -86,10 +96,10 @@ func (self *ApierV1) GetTPActionPlanIds(attrs AttrGetTPActionPlanIds, reply *[]s
 // Removes specific ActionPlan on Tariff plan
 func (self *ApierV1) RemTPActionPlan(attrs AttrGetTPActionPlan, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Id"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_ACTION_PLANS, attrs.TPid, attrs.Id); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_ACTION_PLANS, attrs.TPid, attrs.Id); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tpactions.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,19 +16,19 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"errors"
 	"fmt"
 
+	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
 // Creates a new Actions profile within a tariff plan
 func (self *ApierV1) SetTPActions(attrs utils.TPActions, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionsId", "Actions"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	for _, action := range attrs.Actions {
 		requiredFields := []string{"Identifier", "Weight"}
@@ -36,11 +36,12 @@ func (self *ApierV1) SetTPActions(attrs utils.TPActions, reply *string) error {
 			requiredFields = append(requiredFields, "Direction", "Units")
 		}
 		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
-			return fmt.Errorf("%s:Action:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
+			return fmt.Errorf("%s:Action:%s:%v", utils.ErrMandatoryIeMissing.Error(), action.Identifier, missing)
 		}
 	}
-	if err := self.StorDb.SetTPActions(attrs.TPid, map[string][]*utils.TPAction{attrs.ActionsId: attrs.Actions}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	as := engine.APItoModelAction(&attrs)
+	if err := self.StorDb.SetTpActions(as); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -54,31 +55,36 @@ type AttrGetTPActions struct {
 // Queries specific Actions profile on tariff plan
 func (self *ApierV1) GetTPActions(attrs AttrGetTPActions, reply *utils.TPActions) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionsId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	if acts, err := self.StorDb.GetTpActions(attrs.TPid, attrs.ActionsId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	} else if len(acts) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
-		*reply = utils.TPActions{TPid: attrs.TPid, ActionsId: attrs.ActionsId, Actions: acts[attrs.ActionsId]}
+		as, err := engine.TpActions(acts).GetActions()
+		if err != nil {
+
+		}
+		*reply = utils.TPActions{TPid: attrs.TPid, ActionsId: attrs.ActionsId, Actions: as[attrs.ActionsId]}
 	}
 	return nil
 }
 
 type AttrGetTPActionIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries Actions identities on specific tariff plan.
 func (self *ApierV1) GetTPActionIds(attrs AttrGetTPActionIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_ACTIONS, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACTIONS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -88,10 +94,10 @@ func (self *ApierV1) GetTPActionIds(attrs AttrGetTPActionIds, reply *[]string) e
 // Removes specific Actions on Tariff plan
 func (self *ApierV1) RemTPActions(attrs AttrGetTPActions, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionsId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_ACTIONS, attrs.TPid, attrs.ActionsId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_ACTIONS, attrs.TPid, attrs.ActionsId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tpactiontriggers.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,12 +16,10 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"errors"
-	"fmt"
-
+	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
@@ -29,13 +27,12 @@ import (
 func (self *ApierV1) SetTPActionTriggers(attrs utils.TPActionTriggers, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs,
 		[]string{"TPid", "ActionTriggersId"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	ats := map[string][]*utils.TPActionTrigger{
-		attrs.ActionTriggersId: attrs.ActionTriggers}
 
-	if err := self.StorDb.SetTPActionTriggers(attrs.TPid, ats); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	ats := engine.APItoModelActionTrigger(&attrs)
+	if err := self.StorDb.SetTpActionTriggers(ats); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -49,14 +46,22 @@ type AttrGetTPActionTriggers struct {
 // Queries specific ActionTriggers profile on tariff plan
 func (self *ApierV1) GetTPActionTriggers(attrs AttrGetTPActionTriggers, reply *utils.TPActionTriggers) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionTriggersId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if atsMap, err := self.StorDb.GetTpActionTriggers(attrs.TPid, attrs.ActionTriggersId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if len(atsMap) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+	if ats, err := self.StorDb.GetTpActionTriggers(attrs.TPid, attrs.ActionTriggersId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(ats) == 0 {
+		return utils.ErrNotFound
 	} else {
-		atRply := &utils.TPActionTriggers{attrs.TPid, attrs.ActionTriggersId, atsMap[attrs.ActionTriggersId]}
+		atsMap, err := engine.TpActionTriggers(ats).GetActionTriggers()
+		if err != nil {
+			return err
+		}
+		atRply := &utils.TPActionTriggers{
+			TPid:             attrs.TPid,
+			ActionTriggersId: attrs.ActionTriggersId,
+			ActionTriggers:   atsMap[attrs.ActionTriggersId],
+		}
 		*reply = *atRply
 	}
 	return nil
@@ -64,17 +69,18 @@ func (self *ApierV1) GetTPActionTriggers(attrs AttrGetTPActionTriggers, reply *u
 
 type AttrGetTPActionTriggerIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries ActionTriggers identities on specific tariff plan.
 func (self *ApierV1) GetTPActionTriggerIds(attrs AttrGetTPActionTriggerIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_ACTION_TRIGGERS, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACTION_TRIGGERS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -84,10 +90,10 @@ func (self *ApierV1) GetTPActionTriggerIds(attrs AttrGetTPActionTriggerIds, repl
 // Removes specific ActionTriggers on Tariff plan
 func (self *ApierV1) RemTPActionTriggers(attrs AttrGetTPActionTriggers, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionTriggersId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_ACTION_TRIGGERS, attrs.TPid, attrs.ActionTriggersId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_ACTION_TRIGGERS, attrs.TPid, attrs.ActionTriggersId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tpcdrstats.go

@@ -0,0 +1,103 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new CdrStats profile within a tariff plan
+func (self *ApierV1) SetTPCdrStats(attrs utils.TPCdrStats, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "CdrStatsId", "CdrStats"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	/*for _, action := range attrs.CdrStats {
+		requiredFields := []string{"Identifier", "Weight"}
+		if action.BalanceType != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions
+			requiredFields = append(requiredFields, "Direction", "Units")
+		}
+		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
+			return fmt.Errorf("%s:CdrStat:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
+		}
+	}*/
+	cs := engine.APItoModelCdrStat(&attrs)
+	if err := self.StorDb.SetTpCdrStats(cs); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPCdrStats struct {
+	TPid       string // Tariff plan id
+	CdrStatsId string // CdrStat id
+}
+
+// Queries specific CdrStat on tariff plan
+func (self *ApierV1) GetTPCdrStats(attrs AttrGetTPCdrStats, reply *utils.TPCdrStats) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "CdrStatsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if sgs, err := self.StorDb.GetTpCdrStats(attrs.TPid, attrs.CdrStatsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(sgs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		csMap, err := engine.TpCdrStats(sgs).GetCdrStats()
+		if err != nil {
+			return err
+		}
+		*reply = utils.TPCdrStats{TPid: attrs.TPid, CdrStatsId: attrs.CdrStatsId, CdrStats: csMap[attrs.CdrStatsId]}
+	}
+	return nil
+}
+
+type AttrGetTPCdrStatIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries CdrStats identities on specific tariff plan.
+func (self *ApierV1) GetTPCdrStatsIds(attrs AttrGetTPCdrStatIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_CDR_STATS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific CdrStats on Tariff plan
+func (self *ApierV1) RemTPCdrStats(attrs AttrGetTPCdrStats, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "CdrStatsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_SHARED_GROUPS, attrs.TPid, attrs.CdrStatsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpderivedcharges.go

@@ -0,0 +1,112 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new DerivedCharges profile within a tariff plan
+func (self *ApierV1) SetTPDerivedChargers(attrs utils.TPDerivedChargers, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Direction", "Tenant", "Category", "Account", "Subject"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	/*for _, action := range attrs.DerivedCharges {
+		requiredFields := []string{"Identifier", "Weight"}
+		if action.BalanceType != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions
+			requiredFields = append(requiredFields, "Direction", "Units")
+		}
+		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
+			return fmt.Errorf("%s:DerivedCharge:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
+		}
+	}*/
+	dc := engine.APItoModelDerivedCharger(&attrs)
+	if err := self.StorDb.SetTpDerivedChargers(dc); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPDerivedChargers struct {
+	TPid              string // Tariff plan id
+	DerivedChargersId string // DerivedCharge id
+}
+
+// Queries specific DerivedCharge on tariff plan
+func (self *ApierV1) GetTPDerivedChargers(attrs AttrGetTPDerivedChargers, reply *utils.TPDerivedChargers) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DerivedChargersId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpDc := &utils.TPDerivedChargers{TPid: attrs.TPid}
+	if err := tmpDc.SetDerivedChargersId(attrs.DerivedChargersId); err != nil {
+		return err
+	}
+	dcs := engine.APItoModelDerivedCharger(tmpDc)
+	if sgs, err := self.StorDb.GetTpDerivedChargers(&dcs[0]); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(sgs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		dcsMap, err := engine.TpDerivedChargers(dcs).GetDerivedChargers()
+		if err != nil {
+			return err
+		}
+		*reply = *dcsMap[attrs.DerivedChargersId]
+	}
+	return nil
+}
+
+type AttrGetTPDerivedChargeIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries DerivedCharges identities on specific tariff plan.
+func (self *ApierV1) GetTPDerivedChargerIds(attrs AttrGetTPDerivedChargeIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_DERIVED_CHARGERS, utils.TPDistinctIds{"loadid", "direction", "tenant", "category", "account", "subject"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific DerivedCharges on Tariff plan
+func (self *ApierV1) RemTPDerivedChargers(attrs AttrGetTPDerivedChargers, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DerivedChargesId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpDc := engine.TpDerivedCharger{}
+	if err := tmpDc.SetDerivedChargersId(attrs.DerivedChargersId); err != nil {
+		return err
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_DERIVED_CHARGERS, attrs.TPid, tmpDc.Loadid, tmpDc.Direction, tmpDc.Tenant, tmpDc.Category, tmpDc.Account, tmpDc.Subject); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpdestinationrates.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,24 +16,23 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 // This file deals with tp_destination_rates management over APIs
 
 import (
-	"errors"
-	"fmt"
-
+	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
 // Creates a new DestinationRate profile within a tariff plan
 func (self *ApierV1) SetTPDestinationRate(attrs utils.TPDestinationRate, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationRateId", "DestinationRates"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.SetTPDestinationRates(attrs.TPid, map[string][]*utils.DestinationRate{attrs.DestinationRateId: attrs.DestinationRates}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	drs := engine.APItoModelDestinationRate(&attrs)
+	if err := self.StorDb.SetTpDestinationRates(drs); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -42,36 +41,42 @@ func (self *ApierV1) SetTPDestinationRate(attrs utils.TPDestinationRate, reply *
 type AttrGetTPDestinationRate struct {
 	TPid              string // Tariff plan id
 	DestinationRateId string // Rate id
+	utils.Paginator
 }
 
 // Queries specific DestinationRate profile on tariff plan
 func (self *ApierV1) GetTPDestinationRate(attrs AttrGetTPDestinationRate, reply *utils.TPDestinationRate) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationRateId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if drs, err := self.StorDb.GetTpDestinationRates(attrs.TPid, attrs.DestinationRateId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if drs, err := self.StorDb.GetTpDestinationRates(attrs.TPid, attrs.DestinationRateId, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if len(drs) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
-		*reply = *drs[attrs.DestinationRateId]
+		drsMap, err := engine.TpDestinationRates(drs).GetDestinationRates()
+		if err != nil {
+			return err
+		}
+		*reply = *drsMap[attrs.DestinationRateId]
 	}
 	return nil
 }
 
 type AttrTPDestinationRateIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries DestinationRate identities on specific tariff plan.
 func (self *ApierV1) GetTPDestinationRateIds(attrs AttrGetTPRateIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_DESTINATION_RATES, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_DESTINATION_RATES, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -81,10 +86,10 @@ func (self *ApierV1) GetTPDestinationRateIds(attrs AttrGetTPRateIds, reply *[]st
 // Removes specific DestinationRate on Tariff plan
 func (self *ApierV1) RemTPDestinationRate(attrs AttrGetTPDestinationRate, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationRateId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_DESTINATION_RATES, attrs.TPid, attrs.DestinationRateId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_DESTINATION_RATES, attrs.TPid, attrs.DestinationRateId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tpdestinations.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,12 +16,9 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"errors"
-	"fmt"
-
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
@@ -29,10 +26,11 @@ import (
 // Creates a new destination within a tariff plan
 func (self *ApierV1) SetTPDestination(attrs utils.TPDestination, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationId", "Prefixes"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.SetTPDestination(attrs.TPid, &engine.Destination{Id: attrs.DestinationId, Prefixes: attrs.Prefixes}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	ds := engine.APItoModelDestination(&attrs)
+	if err := self.StorDb.SetTpDestinations(ds); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -46,31 +44,40 @@ type AttrGetTPDestination struct {
 // Queries a specific destination
 func (self *ApierV1) GetTPDestination(attrs AttrGetTPDestination, reply *utils.TPDestination) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if dsts, err := self.StorDb.GetTpDestinations(attrs.TPid, attrs.DestinationId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if len(dsts) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+	if storData, err := self.StorDb.GetTpDestinations(attrs.TPid, attrs.DestinationId); err != nil {
+
+		return utils.NewErrServerError(err)
+	} else if len(storData) == 0 {
+		return utils.ErrNotFound
 	} else {
-		*reply = utils.TPDestination{attrs.TPid, dsts[attrs.DestinationId].Id, dsts[attrs.DestinationId].Prefixes}
+		dsts, err := engine.TpDestinations(storData).GetDestinations()
+		if err != nil {
+			return err
+		}
+		*reply = utils.TPDestination{
+			TPid:          attrs.TPid,
+			DestinationId: dsts[attrs.DestinationId].Id,
+			Prefixes:      dsts[attrs.DestinationId].Prefixes}
 	}
 	return nil
 }
 
 type AttrGetTPDestinationIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries destination identities on specific tariff plan.
 func (self *ApierV1) GetTPDestinationIds(attrs AttrGetTPDestinationIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_DESTINATIONS, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_DESTINATIONS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -79,10 +86,10 @@ func (self *ApierV1) GetTPDestinationIds(attrs AttrGetTPDestinationIds, reply *[
 
 func (self *ApierV1) RemTPDestination(attrs AttrGetTPDestination, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_DESTINATIONS, attrs.TPid, attrs.DestinationId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_DESTINATIONS, attrs.TPid, attrs.DestinationId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tplcrrules.go

@@ -0,0 +1,92 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+/*
+// Creates a new LcrRules profile within a tariff plan
+func (self *ApierV1) SetTPLcrRules(attrs utils.TPLcrRules, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LcrRulesId", "LcrRules"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	for _, action := range attrs.LcrRules {
+		requiredFields := []string{"Identifier", "Weight"}
+
+		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
+			return fmt.Errorf("%s:LcrAction:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
+		}
+	}
+	if err := self.StorDb.SetTPLcrRules(attrs.TPid, map[string][]*utils.TPLcrRule{attrs.LcrRulesId: attrs.LcrRules}); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPLcrRules struct {
+	TPid  string // Tariff plan id
+	LcrId string // Lcr id
+}
+
+// Queries specific LcrRules profile on tariff plan
+func (self *ApierV1) GetTPLcrRules(attrs AttrGetTPLcrRules, reply *utils.TPLcrRules) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LcrId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if lcrs, err := self.StorDb.GetTpLCRs(attrs.TPid, attrs.LcrId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(acts) == 0 {
+		return utils.ErrNotFound
+	} else {
+		*reply = utils.TPLcrRules{TPid: attrs.TPid, LcrRulesId: attrs.LcrRulesId, LcrRules: lcrs[attrs.LcrRulesId]}
+	}
+	return nil
+}
+
+type AttrGetTPLcrActionIds struct {
+	TPid string // Tariff plan id
+}
+
+// Queries LcrRules identities on specific tariff plan.
+func (self *ApierV1) GetTPLcrActionIds(attrs AttrGetTPLcrActionIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_LCRS, "id", nil); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific LcrRules on Tariff plan
+func (self *ApierV1) RemTPLcrRules(attrs AttrGetTPLcrRules, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LcrRulesId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTPData(utils.TBL_TP_LCRS, attrs.TPid, attrs.LcrRulesId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}
+*/

apier/v1/tprates.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,24 +16,23 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 // This file deals with tp_rates management over APIs
 
 import (
-	"errors"
-	"fmt"
-
+	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
 // Creates a new rate within a tariff plan
 func (self *ApierV1) SetTPRate(attrs utils.TPRate, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RateId", "RateSlots"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.SetTPRates(attrs.TPid, map[string][]*utils.RateSlot{attrs.RateId: attrs.RateSlots}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	r := engine.APItoModelRate(&attrs)
+	if err := self.StorDb.SetTpRates(r); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -47,31 +46,36 @@ type AttrGetTPRate struct {
 // Queries specific Rate on tariff plan
 func (self *ApierV1) GetTPRate(attrs AttrGetTPRate, reply *utils.TPRate) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RateId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	if rts, err := self.StorDb.GetTpRates(attrs.TPid, attrs.RateId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	} else if len(rts) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
-		*reply = *rts[attrs.RateId]
+		rtsMap, err := engine.TpRates(rts).GetRates()
+		if err != nil {
+			return err
+		}
+		*reply = *rtsMap[attrs.RateId]
 	}
 	return nil
 }
 
 type AttrGetTPRateIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries rate identities on specific tariff plan.
 func (self *ApierV1) GetTPRateIds(attrs AttrGetTPRateIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_RATES, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_RATES, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -81,10 +85,10 @@ func (self *ApierV1) GetTPRateIds(attrs AttrGetTPRateIds, reply *[]string) error
 // Removes specific Rate on Tariff plan
 func (self *ApierV1) RemTPRate(attrs AttrGetTPRate, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RateId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_RATES, attrs.TPid, attrs.RateId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_RATES, attrs.TPid, attrs.RateId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tpratingplans.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,24 +16,23 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 // This file deals with tp_destrates_timing management over APIs
 
 import (
-	"errors"
-	"fmt"
-
+	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
 // Creates a new DestinationRateTiming profile within a tariff plan
 func (self *ApierV1) SetTPRatingPlan(attrs utils.TPRatingPlan, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingPlanId", "RatingPlanBindings"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.SetTPRatingPlans(attrs.TPid, map[string][]*utils.TPRatingPlanBinding{attrs.RatingPlanId: attrs.RatingPlanBindings}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	rp := engine.APItoModelRatingPlan(&attrs)
+	if err := self.StorDb.SetTpRatingPlans(rp); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -42,36 +41,42 @@ func (self *ApierV1) SetTPRatingPlan(attrs utils.TPRatingPlan, reply *string) er
 type AttrGetTPRatingPlan struct {
 	TPid         string // Tariff plan id
 	RatingPlanId string // Rate id
+	utils.Paginator
 }
 
 // Queries specific RatingPlan profile on tariff plan
 func (self *ApierV1) GetTPRatingPlan(attrs AttrGetTPRatingPlan, reply *utils.TPRatingPlan) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingPlanId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if rps, err := self.StorDb.GetTpRatingPlans(attrs.TPid, attrs.RatingPlanId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if rps, err := self.StorDb.GetTpRatingPlans(attrs.TPid, attrs.RatingPlanId, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if len(rps) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
-		*reply = utils.TPRatingPlan{TPid: attrs.TPid, RatingPlanId: attrs.RatingPlanId, RatingPlanBindings: rps[attrs.RatingPlanId]}
+		rpsMap, err := engine.TpRatingPlans(rps).GetRatingPlans()
+		if err != nil {
+			return err
+		}
+		*reply = utils.TPRatingPlan{TPid: attrs.TPid, RatingPlanId: attrs.RatingPlanId, RatingPlanBindings: rpsMap[attrs.RatingPlanId]}
 	}
 	return nil
 }
 
 type AttrGetTPRatingPlanIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries RatingPlan identities on specific tariff plan.
 func (self *ApierV1) GetTPRatingPlanIds(attrs AttrGetTPRatingPlanIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_RATING_PLANS, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_RATING_PLANS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -81,10 +86,10 @@ func (self *ApierV1) GetTPRatingPlanIds(attrs AttrGetTPRatingPlanIds, reply *[]s
 // Removes specific RatingPlan on Tariff plan
 func (self *ApierV1) RemTPRatingPlan(attrs AttrGetTPRatingPlan, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingPlanId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_RATING_PLANS, attrs.TPid, attrs.RatingPlanId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_RATING_PLANS, attrs.TPid, attrs.RatingPlanId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tpratingprofiles.go

@@ -0,0 +1,170 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+// This file deals with tp_rate_profiles management over APIs
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new RatingProfile within a tariff plan
+func (self *ApierV1) SetTPRatingProfile(attrs utils.TPRatingProfile, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "Category", "Direction", "Subject", "RatingPlanActivations"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	rpf := engine.APItoModelRatingProfile(&attrs)
+	if err := self.StorDb.SetTpRatingProfiles(rpf); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPRatingProfileByLoadId struct {
+	TPid   string // Tariff plan id
+	LoadId string // RatingProfile id
+}
+
+// Queries specific RatingProfile on tariff plan
+func (self *ApierV1) GetTPRatingProfilesByLoadId(attrs utils.TPRatingProfile, reply *[]*utils.TPRatingProfile) error {
+	mndtryFlds := []string{"TPid", "LoadId"}
+	if len(attrs.Subject) != 0 { // If Subject provided as filter, make all related fields mandatory
+		mndtryFlds = append(mndtryFlds, "Tenant", "TOR", "Direction", "Subject")
+	}
+	if missing := utils.MissingStructFields(&attrs, mndtryFlds); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	rpf := engine.APItoModelRatingProfile(&attrs)
+	if dr, err := self.StorDb.GetTpRatingProfiles(&rpf[0]); err != nil {
+		return utils.NewErrServerError(err)
+	} else if dr == nil {
+		return utils.ErrNotFound
+	} else {
+		rpfMap, err := engine.TpRatingProfiles(dr).GetRatingProfiles()
+		if err != nil {
+			return err
+		}
+		var rpfs []*utils.TPRatingProfile
+		if len(attrs.Subject) != 0 {
+			rpfs = []*utils.TPRatingProfile{rpfMap[attrs.KeyId()]}
+		} else {
+			for _, rpfLst := range rpfMap {
+				rpfs = append(rpfs, rpfLst)
+			}
+		}
+		*reply = rpfs
+	}
+	return nil
+}
+
+// Queries RatingProfile identities on specific tariff plan.
+func (self *ApierV1) GetTPRatingProfileLoadIds(attrs utils.AttrTPRatingProfileIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_RATE_PROFILES, utils.TPDistinctIds{"loadid"}, map[string]string{
+		"tenant":    attrs.Tenant,
+		"tor":       attrs.Category,
+		"direction": attrs.Direction,
+		"subject":   attrs.Subject,
+	}, new(utils.Paginator)); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+type AttrGetTPRatingProfile struct {
+	TPid            string // Tariff plan id
+	RatingProfileId string // RatingProfile id
+}
+
+// Queries specific RatingProfile on tariff plan
+func (self *ApierV1) GetTPRatingProfile(attrs AttrGetTPRatingProfile, reply *utils.TPRatingProfile) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingProfileId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpRpf := &utils.TPRatingProfile{TPid: attrs.TPid}
+	if err := tmpRpf.SetRatingProfilesId(attrs.RatingProfileId); err != nil {
+		return err
+	}
+	rpf := engine.APItoModelRatingProfile(tmpRpf)
+	if rpfs, err := self.StorDb.GetTpRatingProfiles(&rpf[0]); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(rpfs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		rpfMap, err := engine.TpRatingProfiles(rpfs).GetRatingProfiles()
+		if err != nil {
+			return err
+		}
+		rpf := rpfMap[tmpRpf.KeyId()]
+		tpdc := utils.TPRatingProfile{
+			TPid: attrs.TPid,
+			RatingPlanActivations: rpf.RatingPlanActivations,
+		}
+		if err := tpdc.SetRatingProfilesId(attrs.RatingProfileId); err != nil {
+			return err
+		}
+		*reply = tpdc
+	}
+	return nil
+}
+
+type AttrGetTPRatingProfileIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries RatingProfiles identities on specific tariff plan.
+func (self *ApierV1) GetTPRatingProfileIds(attrs AttrGetTPRatingProfileIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_RATE_PROFILES, utils.TPDistinctIds{"loadid", "direction", "tenant", "category", "subject"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific RatingProfiles on Tariff plan
+func (self *ApierV1) RemTPRatingProfile(attrs AttrGetTPRatingProfile, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingProfileId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpRpf := engine.TpRatingProfile{}
+	if err := tmpRpf.SetRatingProfileId(attrs.RatingProfileId); err != nil {
+		return err
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_RATE_PROFILES, attrs.TPid, tmpRpf.Loadid, tmpRpf.Direction, tmpRpf.Tenant, tmpRpf.Category, tmpRpf.Subject); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpsharedgroups.go

@@ -0,0 +1,103 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new SharedGroups profile within a tariff plan
+func (self *ApierV1) SetTPSharedGroups(attrs utils.TPSharedGroups, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "SharedGroupsId", "SharedGroups"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	/*for _, action := range attrs.SharedGroups {
+		requiredFields := []string{"Identifier", "Weight"}
+		if action.BalanceType != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions
+			requiredFields = append(requiredFields, "Direction", "Units")
+		}
+		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
+			return fmt.Errorf("%s:SharedGroup:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
+		}
+	}*/
+	sg := engine.APItoModelSharedGroup(&attrs)
+	if err := self.StorDb.SetTpSharedGroups(sg); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPSharedGroups struct {
+	TPid           string // Tariff plan id
+	SharedGroupsId string // SharedGroup id
+}
+
+// Queries specific SharedGroup on tariff plan
+func (self *ApierV1) GetTPSharedGroups(attrs AttrGetTPSharedGroups, reply *utils.TPSharedGroups) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "SharedGroupsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if sgs, err := self.StorDb.GetTpSharedGroups(attrs.TPid, attrs.SharedGroupsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(sgs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		sgMap, err := engine.TpSharedGroups(sgs).GetSharedGroups()
+		if err != nil {
+			return err
+		}
+		*reply = utils.TPSharedGroups{TPid: attrs.TPid, SharedGroupsId: attrs.SharedGroupsId, SharedGroups: sgMap[attrs.SharedGroupsId]}
+	}
+	return nil
+}
+
+type AttrGetTPSharedGroupIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries SharedGroups identities on specific tariff plan.
+func (self *ApierV1) GetTPSharedGroupIds(attrs AttrGetTPSharedGroupIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_SHARED_GROUPS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific SharedGroups on Tariff plan
+func (self *ApierV1) RemTPSharedGroups(attrs AttrGetTPSharedGroups, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "SharedGroupsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_SHARED_GROUPS, attrs.TPid, attrs.SharedGroupsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tptimings.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,12 +16,9 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"errors"
-	"fmt"
-
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
@@ -29,11 +26,11 @@ import (
 // Creates a new timing within a tariff plan
 func (self *ApierV1) SetTPTiming(attrs utils.ApierTPTiming, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "TimingId", "Years", "Months", "MonthDays", "WeekDays", "Time"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	tm := engine.NewTiming(attrs.TimingId, attrs.Years, attrs.Months, attrs.MonthDays, attrs.WeekDays, attrs.Time)
-	if err := self.StorDb.SetTPTiming(attrs.TPid, tm); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	tm := engine.APItoModelTiming(&attrs)
+	if err := self.StorDb.SetTpTimings([]engine.TpTiming{*tm}); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -47,33 +44,36 @@ type AttrGetTPTiming struct {
 // Queries specific Timing on Tariff plan
 func (self *ApierV1) GetTPTiming(attrs AttrGetTPTiming, reply *utils.ApierTPTiming) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "TimingId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	if tms, err := self.StorDb.GetTpTimings(attrs.TPid, attrs.TimingId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	} else if len(tms) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
-		tm := tms[attrs.TimingId]
-		*reply = utils.ApierTPTiming{attrs.TPid, tm.Id, tm.Years.Serialize(";"),
-			tm.Months.Serialize(";"), tm.MonthDays.Serialize(";"), tm.WeekDays.Serialize(";"), tm.StartTime}
+		tmMap, err := engine.TpTimings(tms).GetApierTimings()
+		if err != nil {
+			return err
+		}
+		*reply = *tmMap[attrs.TimingId]
 	}
 	return nil
 }
 
 type AttrGetTPTimingIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries timing identities on specific tariff plan.
 func (self *ApierV1) GetTPTimingIds(attrs AttrGetTPTimingIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_TIMINGS, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_TIMINGS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -83,10 +83,10 @@ func (self *ApierV1) GetTPTimingIds(attrs AttrGetTPTimingIds, reply *[]string) e
 // Removes specific Timing on Tariff plan
 func (self *ApierV1) RemTPTiming(attrs AttrGetTPTiming, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "TimingId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_TIMINGS, attrs.TPid, attrs.TimingId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_TIMINGS, attrs.TPid, attrs.TimingId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tutfsjson_local_test

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

apier/v2/apier.go

@@ -0,0 +1,122 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"github.com/cgrates/cgrates/apier/v1"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type ApierV2 struct {
+	v1.ApierV1
+}
+
+type AttrLoadRatingProfile struct {
+	TPid            string
+	RatingProfileId string
+}
+
+// Process dependencies and load a specific rating profile from storDb into dataDb.
+func (self *ApierV2) LoadRatingProfile(attrs AttrLoadRatingProfile, reply *string) error {
+	if len(attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	tpRpf := &utils.TPRatingProfile{TPid: attrs.TPid}
+	tpRpf.SetRatingProfilesId(attrs.RatingProfileId)
+	rpf := engine.APItoModelRatingProfile(tpRpf)
+	dbReader := engine.NewTpReader(self.RatingDb, self.AccountDb, self.StorDb, attrs.TPid)
+	if err := dbReader.LoadRatingProfilesFiltered(&rpf[0]); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	//Automatic cache of the newly inserted rating profile
+	var ratingProfile []string
+	if tpRpf.KeyId() != ":::" { // if has some filters
+		ratingProfile = []string{utils.RATING_PROFILE_PREFIX + tpRpf.KeyId()}
+	}
+	if err := self.RatingDb.CachePrefixValues(map[string][]string{utils.RATING_PROFILE_PREFIX: ratingProfile}); err != nil {
+		return err
+	}
+	*reply = v1.OK
+	return nil
+}
+
+type AttrLoadAccountActions struct {
+	TPid             string
+	AccountActionsId string
+}
+
+// Process dependencies and load a specific AccountActions profile from storDb into dataDb.
+func (self *ApierV2) LoadAccountActions(attrs AttrLoadAccountActions, reply *string) error {
+	if len(attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	dbReader := engine.NewTpReader(self.RatingDb, self.AccountDb, self.StorDb, attrs.TPid)
+	tpAa := &utils.TPAccountActions{TPid: attrs.TPid}
+	tpAa.SetAccountActionsId(attrs.AccountActionsId)
+	aa := engine.APItoModelAccountAction(tpAa)
+	if _, err := engine.Guardian.Guard(func() (interface{}, error) {
+		if err := dbReader.LoadAccountActionsFiltered(aa); err != nil {
+			return 0, err
+		}
+		return 0, nil
+	}, attrs.AccountActionsId); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	// ToDo: Get the action keys loaded by dbReader so we reload only these in cache
+	// Need to do it before scheduler otherwise actions to run will be unknown
+	if err := self.RatingDb.CachePrefixes(utils.DERIVED_CHARGERS_CSV, utils.ACTION_PREFIX, utils.SHARED_GROUP_PREFIX, utils.ACC_ALIAS_PREFIX); err != nil {
+		return err
+	}
+	if self.Sched != nil {
+		self.Sched.LoadActionPlans(self.RatingDb)
+		self.Sched.Restart()
+	}
+	*reply = v1.OK
+	return nil
+}
+
+type AttrLoadDerivedChargers struct {
+	TPid              string
+	DerivedChargersId string
+}
+
+// Load derived chargers from storDb into dataDb.
+func (self *ApierV2) LoadDerivedChargers(attrs AttrLoadDerivedChargers, reply *string) error {
+	if len(attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	tpDc := &utils.TPDerivedChargers{TPid: attrs.TPid}
+	tpDc.SetDerivedChargersId(attrs.DerivedChargersId)
+	dc := engine.APItoModelDerivedCharger(tpDc)
+	dbReader := engine.NewTpReader(self.RatingDb, self.AccountDb, self.StorDb, attrs.TPid)
+	if err := dbReader.LoadDerivedChargersFiltered(&dc[0], true); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	//Automatic cache of the newly inserted rating plan
+	var dcsChanged []string
+	if len(attrs.DerivedChargersId) != 0 {
+		dcsChanged = []string{utils.DERIVEDCHARGERS_PREFIX + attrs.DerivedChargersId}
+	}
+	if err := self.RatingDb.CachePrefixValues(map[string][]string{utils.DERIVEDCHARGERS_PREFIX: dcsChanged}); err != nil {
+		return err
+	}
+	*reply = v1.OK
+	return nil
+}

apier/v2/cdre.go

@@ -0,0 +1,134 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/cgrates/cgrates/cdre"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Export Cdrs to file
+func (self *ApierV2) ExportCdrsToFile(attr utils.AttrExportCdrsToFile, reply *utils.ExportedFileCdrs) error {
+	var err error
+	exportTemplate := self.Config.CdreProfiles[utils.META_DEFAULT]
+	if attr.ExportTemplate != nil && len(*attr.ExportTemplate) != 0 { // Export template prefered, use it
+		var hasIt bool
+		if exportTemplate, hasIt = self.Config.CdreProfiles[*attr.ExportTemplate]; !hasIt {
+			return fmt.Errorf("%s:ExportTemplate", utils.ErrNotFound)
+		}
+	}
+	cdrFormat := exportTemplate.CdrFormat
+	if attr.CdrFormat != nil && len(*attr.CdrFormat) != 0 {
+		cdrFormat = strings.ToLower(*attr.CdrFormat)
+	}
+	if !utils.IsSliceMember(utils.CdreCdrFormats, cdrFormat) {
+		return utils.NewErrMandatoryIeMissing("CdrFormat")
+	}
+	fieldSep := exportTemplate.FieldSeparator
+	if attr.FieldSeparator != nil && len(*attr.FieldSeparator) != 0 {
+		fieldSep, _ = utf8.DecodeRuneInString(*attr.FieldSeparator)
+		if fieldSep == utf8.RuneError {
+			return fmt.Errorf("%s:FieldSeparator:%s", utils.ErrServerError, "Invalid")
+		}
+	}
+	exportDir := exportTemplate.ExportDir
+	if attr.ExportDir != nil && len(*attr.ExportDir) != 0 {
+		exportDir = *attr.ExportDir
+	}
+	exportId := strconv.FormatInt(time.Now().Unix(), 10)
+	if attr.ExportId != nil && len(*attr.ExportId) != 0 {
+		exportId = *attr.ExportId
+	}
+	fileName := fmt.Sprintf("cdre_%s.%s", exportId, cdrFormat)
+	if attr.ExportFileName != nil && len(*attr.ExportFileName) != 0 {
+		fileName = *attr.ExportFileName
+	}
+	filePath := path.Join(exportDir, fileName)
+	if cdrFormat == utils.DRYRUN {
+		filePath = utils.DRYRUN
+	}
+	dataUsageMultiplyFactor := exportTemplate.DataUsageMultiplyFactor
+	if attr.DataUsageMultiplyFactor != nil && *attr.DataUsageMultiplyFactor != 0.0 {
+		dataUsageMultiplyFactor = *attr.DataUsageMultiplyFactor
+	}
+	smsUsageMultiplyFactor := exportTemplate.SmsUsageMultiplyFactor
+	if attr.SmsUsageMultiplyFactor != nil && *attr.SmsUsageMultiplyFactor != 0.0 {
+		smsUsageMultiplyFactor = *attr.SmsUsageMultiplyFactor
+	}
+	genericUsageMultiplyFactor := exportTemplate.GenericUsageMultiplyFactor
+	if attr.GenericUsageMultiplyFactor != nil && *attr.GenericUsageMultiplyFactor != 0.0 {
+		genericUsageMultiplyFactor = *attr.GenericUsageMultiplyFactor
+	}
+	costMultiplyFactor := exportTemplate.CostMultiplyFactor
+	if attr.CostMultiplyFactor != nil && *attr.CostMultiplyFactor != 0.0 {
+		costMultiplyFactor = *attr.CostMultiplyFactor
+	}
+	costShiftDigits := exportTemplate.CostShiftDigits
+	if attr.CostShiftDigits != nil {
+		costShiftDigits = *attr.CostShiftDigits
+	}
+	roundingDecimals := exportTemplate.CostRoundingDecimals
+	if attr.RoundDecimals != nil {
+		roundingDecimals = *attr.RoundDecimals
+	}
+	maskDestId := exportTemplate.MaskDestId
+	if attr.MaskDestinationId != nil && len(*attr.MaskDestinationId) != 0 {
+		maskDestId = *attr.MaskDestinationId
+	}
+	maskLen := exportTemplate.MaskLength
+	if attr.MaskLength != nil {
+		maskLen = *attr.MaskLength
+	}
+	cdrsFltr, err := attr.RpcCdrsFilter.AsCdrsFilter()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	cdrs, _, err := self.CdrDb.GetStoredCdrs(cdrsFltr)
+	if err != nil {
+		return err
+	} else if len(cdrs) == 0 {
+		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
+		return nil
+	}
+	cdrexp, err := cdre.NewCdrExporter(cdrs, self.CdrDb, exportTemplate, cdrFormat, fieldSep, exportId, dataUsageMultiplyFactor, smsUsageMultiplyFactor, genericUsageMultiplyFactor,
+		costMultiplyFactor, costShiftDigits, roundingDecimals, self.Config.RoundingDecimals, maskDestId, maskLen, self.Config.HttpSkipTlsVerify)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if cdrexp.TotalExportedCdrs() == 0 {
+		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
+		return nil
+	}
+	if err := cdrexp.WriteToFile(filePath); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), TotalCost: cdrexp.TotalCost(), FirstOrderId: cdrexp.FirstOrderId(), LastOrderId: cdrexp.LastOrderId()}
+	if !attr.SuppressCgrIds {
+		reply.ExportedCgrIds = cdrexp.PositiveExports()
+		reply.UnexportedCgrIds = cdrexp.NegativeExports()
+	}
+	return nil
+}

apier/v2/cdrs.go

@@ -0,0 +1,63 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"github.com/cgrates/cgrates/apier/v1"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Retrieves CDRs based on the filters
+func (apier *ApierV2) GetCdrs(attrs utils.RpcCdrsFilter, reply *[]*engine.ExternalCdr) error {
+	cdrsFltr, err := attrs.AsCdrsFilter()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if cdrs, _, err := apier.CdrDb.GetStoredCdrs(cdrsFltr); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(cdrs) == 0 {
+		*reply = make([]*engine.ExternalCdr, 0)
+	} else {
+		for _, cdr := range cdrs {
+			*reply = append(*reply, cdr.AsExternalCdr())
+		}
+	}
+	return nil
+}
+
+func (apier *ApierV2) CountCdrs(attrs utils.RpcCdrsFilter, reply *int64) error {
+	cdrsFltr, err := attrs.AsCdrsFilter()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	cdrsFltr.Count = true
+	if _, count, err := apier.CdrDb.GetStoredCdrs(cdrsFltr); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = count
+	}
+	return nil
+}
+
+// Receive CDRs via RPC methods, not included with APIer because it has way less dependencies and can be standalone
+type CdrsV2 struct {
+	v1.CdrsV1
+}

apier/v2/cdrs_mysql_local_test.go

@@ -0,0 +1,250 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"flag"
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"path"
+	"testing"
+	"time"
+)
+
+var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args
+var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here")
+var waitRater = flag.Int("wait_rater", 500, "Number of miliseconds to wait for rater to start and cache")
+
+var cdrsCfgPath string
+var cdrsCfg *config.CGRConfig
+var cdrsRpc *rpc.Client
+
+func TestV2CdrsMysqlInitConfig(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrsCfgPath = path.Join(*dataDir, "conf", "samples", "cdrsv2mysql")
+	if cdrsCfg, err = config.NewCGRConfigFromFolder(cdrsCfgPath); err != nil {
+		t.Fatal("Got config error: ", err.Error())
+	}
+}
+
+func TestV2CdrsMysqlInitDataDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitDataDb(cdrsCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// InitDb so we can rely on count
+func TestV2CdrsMysqlInitCdrDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitStorDb(cdrsCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestV2CdrsMysqlInjectUnratedCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var mysqlDb *engine.MySQLStorage
+	if d, err := engine.NewMySQLStorage(cdrsCfg.StorDBHost, cdrsCfg.StorDBPort, cdrsCfg.StorDBName, cdrsCfg.StorDBUser, cdrsCfg.StorDBPass,
+		cdrsCfg.StorDBMaxOpenConns, cdrsCfg.StorDBMaxIdleConns); err != nil {
+		t.Error("Error on opening database connection: ", err)
+		return
+	} else {
+		mysqlDb = d.(*engine.MySQLStorage)
+	}
+	strCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()),
+		TOR: utils.VOICE, AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: utils.META_RATED,
+		Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+		SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC),
+		Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
+		MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201}
+	if err := mysqlDb.SetCdr(strCdr1); err != nil {
+		t.Error(err.Error())
+	}
+}
+
+func TestV2CdrsMysqlStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if _, err := engine.StopStartEngine(cdrsCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestV2CdrsMysqlRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrsRpc, err = jsonrpc.Dial("tcp", cdrsCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+// Insert some CDRs
+func TestV2CdrsMysqlProcessCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans", Rated: true,
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("abcdeftg", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("aererfddf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+	}
+	for _, cdr := range cdrs {
+		if err := cdrsRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil {
+			t.Error("Unexpected error: ", err.Error())
+		} else if reply != utils.OK {
+			t.Error("Unexpected reply received: ", reply)
+		}
+	}
+}
+
+func TestV2CdrsMysqlGetCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply []*engine.ExternalCdr
+	req := utils.RpcCdrsFilter{}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 4 {
+		t.Error("Unexpected number of CDRs returned: ", len(reply))
+	}
+	// CDRs with errors
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0), MaxCost: utils.Float64Pointer(0.0)}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 2 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// CDRs Rated
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 3 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// CDRs non rated OR SkipRated
+	req = utils.RpcCdrsFilter{MaxCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 1 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// Skip Errors
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(0.0), MaxCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 2 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+}
+
+func TestV2CdrsMysqlCountCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply int64
+	req := utils.AttrGetCdrs{}
+	if err := cdrsRpc.Call("ApierV2.CountCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if reply != 4 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+}
+
+// Test Prepaid CDRs without previous costs being calculated
+func TestV2CdrsMysqlProcessPrepaidCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans", Rated: true,
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("abcdeftg2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("aererfddf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+	}
+	tStart := time.Now()
+	for _, cdr := range cdrs {
+		if err := cdrsRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil {
+			t.Error("Unexpected error: ", err.Error())
+		} else if reply != utils.OK {
+			t.Error("Unexpected reply received: ", reply)
+		}
+	}
+	if processDur := time.Now().Sub(tStart); processDur > 1*time.Second {
+		t.Error("Unexpected processing time", processDur)
+	}
+}
+
+func TestV2CdrsMysqlKillEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.KillEngine(*waitRater); err != nil {
+		t.Error(err)
+	}
+}

apier/v2/cdrs_psql_local_test.go

@@ -0,0 +1,248 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"os/exec"
+	"path"
+	"testing"
+	"time"
+)
+
+var cdrsPsqlCfgPath string
+var cdrsPsqlCfg *config.CGRConfig
+var cdrsPsqlRpc *rpc.Client
+var cmdEngineCdrPsql *exec.Cmd
+
+func TestV2CdrsPsqlInitConfig(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrsPsqlCfgPath = path.Join(*dataDir, "conf", "samples", "cdrsv2psql")
+	if cdrsPsqlCfg, err = config.NewCGRConfigFromFolder(cdrsPsqlCfgPath); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestV2CdrsPsqlInitDataDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitDataDb(cdrsPsqlCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// InitDb so we can rely on count
+func TestV2CdrsPsqlInitCdrDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitStorDb(cdrsPsqlCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestV2CdrsPsqlInjectUnratedCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var psqlDb *engine.PostgresStorage
+	if d, err := engine.NewPostgresStorage(cdrsPsqlCfg.StorDBHost, cdrsPsqlCfg.StorDBPort, cdrsPsqlCfg.StorDBName, cdrsPsqlCfg.StorDBUser, cdrsPsqlCfg.StorDBPass,
+		cdrsPsqlCfg.StorDBMaxOpenConns, cdrsPsqlCfg.StorDBMaxIdleConns); err != nil {
+		t.Error("Error on opening database connection: ", err)
+		return
+	} else {
+		psqlDb = d.(*engine.PostgresStorage)
+	}
+	strCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()),
+		TOR: utils.VOICE, AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: utils.META_RATED,
+		Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+		SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC),
+		Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
+		MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201}
+	if err := psqlDb.SetCdr(strCdr1); err != nil {
+		t.Error(err.Error())
+	}
+}
+
+func TestV2CdrsPsqlStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	if cmdEngineCdrPsql, err = engine.StartEngine(cdrsPsqlCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestV2CdrsPsqlPsqlRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrsPsqlRpc, err = jsonrpc.Dial("tcp", cdrsPsqlCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+// Insert some CDRs
+func TestV2CdrsPsqlProcessCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans", Rated: true,
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("abcdeftg", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("aererfddf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+	}
+	for _, cdr := range cdrs {
+		if err := cdrsPsqlRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil {
+			t.Error("Unexpected error: ", err.Error())
+		} else if reply != utils.OK {
+			t.Error("Unexpected reply received: ", reply)
+		}
+	}
+}
+
+func TestV2CdrsPsqlGetCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply []*engine.ExternalCdr
+	req := utils.RpcCdrsFilter{}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 4 {
+		t.Error("Unexpected number of CDRs returned: ", len(reply))
+	}
+	// CDRs with errors
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0), MaxCost: utils.Float64Pointer(0.0)}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 2 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// CDRs Rated
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 3 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// CDRs non rated OR SkipRated
+	req = utils.RpcCdrsFilter{MaxCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 1 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// Skip Errors
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(0.0), MaxCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 2 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+}
+
+func TestV2CdrsPsqlCountCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply int64
+	req := utils.AttrGetCdrs{}
+	if err := cdrsPsqlRpc.Call("ApierV2.CountCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if reply != 4 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+}
+
+// Test Prepaid CDRs without previous costs being calculated
+func TestV2CdrsPsqlProcessPrepaidCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans", Rated: true,
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("abcdeftg2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("aererfddf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+	}
+	tStart := time.Now()
+	for _, cdr := range cdrs {
+		if err := cdrsPsqlRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil {
+			t.Error("Unexpected error: ", err.Error())
+		} else if reply != utils.OK {
+			t.Error("Unexpected reply received: ", reply)
+		}
+	}
+	if processDur := time.Now().Sub(tStart); processDur > 1*time.Second {
+		t.Error("Unexpected processing time", processDur)
+	}
+}
+
+func TestV2CdrsPsqlKillEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.KillEngine(*waitRater); err != nil {
+		t.Error(err)
+	}
+}

apier/v2/tp.go

@@ -0,0 +1,99 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"encoding/base64"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type AttrRemTp struct {
+	TPid string
+}
+
+func (self *ApierV2) RemTP(attrs AttrRemTp, reply *string) error {
+	if len(attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	if err := self.StorDb.RemTpData("", attrs.TPid); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}
+
+func (self *ApierV2) ExportTPToFolder(attrs utils.AttrDirExportTP, exported *utils.ExportedTPStats) error {
+	if len(*attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	dir := self.Config.TpExportPath
+	if attrs.ExportPath != nil {
+		dir = *attrs.ExportPath
+	}
+	fileFormat := utils.CSV
+	if attrs.FileFormat != nil {
+		fileFormat = *attrs.FileFormat
+	}
+	sep := ","
+	if attrs.FieldSeparator != nil {
+		sep = *attrs.FieldSeparator
+	}
+	compress := false
+	if attrs.Compress != nil {
+		compress = *attrs.Compress
+	}
+	tpExporter, err := engine.NewTPExporter(self.StorDb, *attrs.TPid, dir, fileFormat, sep, compress)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if err := tpExporter.Run(); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*exported = *tpExporter.ExportStats()
+	}
+
+	return nil
+}
+
+func (self *ApierV2) ExportTPToZipString(attrs utils.AttrDirExportTP, reply *string) error {
+	if len(*attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	dir := ""
+	fileFormat := utils.CSV
+	if attrs.FileFormat != nil {
+		fileFormat = *attrs.FileFormat
+	}
+	sep := ","
+	if attrs.FieldSeparator != nil {
+		sep = *attrs.FieldSeparator
+	}
+	tpExporter, err := engine.NewTPExporter(self.StorDb, *attrs.TPid, dir, fileFormat, sep, true)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if err := tpExporter.Run(); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = base64.StdEncoding.EncodeToString(tpExporter.GetCacheBuffer().Bytes())
+	return nil
+}

cache2go/cache.go

@@ -2,281 +2,176 @@
 package cache2go
 
 import (
-	"errors"
-	"strings"
 	"sync"
 	"time"
 )
 
-type expiringCacheEntry interface {
-	XCache(key string, expire time.Duration, value expiringCacheEntry)
-	timer() *time.Timer
-	age() time.Duration
-	KeepAlive()
-}
-
-// Structure that must be embeded in the objectst that must be cached with expiration.
-// If the expiration is not needed this can be ignored
-type XEntry struct {
-	sync.Mutex
-	key            string
-	keepAlive      bool
-	expireDuration time.Duration
-	timestamp      time.Time
-	t              *time.Timer
-}
+const (
+	PREFIX_LEN   = 4
+	KIND_ADD     = "ADD"
+	KIND_ADP     = "ADP"
+	KIND_REM     = "REM"
+	KIND_PRF     = "PRF"
+	DOUBLE_CACHE = true
+)
 
 type timestampedValue struct {
 	timestamp time.Time
 	value     interface{}
 }
 
-const (
-	PREFIX_LEN = 4
-)
+func (tsv timestampedValue) Value() interface{} {
+	return tsv.value
+}
+
+type transactionItem struct {
+	key   string
+	value interface{}
+	kind  string
+}
+
+func init() {
+	if DOUBLE_CACHE {
+		cache = newDoubleStore()
+	} else {
+		cache = newSimpleStore()
+	}
+}
 
 var (
-	xcache   = make(map[string]expiringCacheEntry)
-	xMux     sync.RWMutex
-	cache    = make(map[string]timestampedValue)
-	mux      sync.RWMutex
-	cMux     sync.Mutex
-	counters = make(map[string]int64)
+	mux   sync.RWMutex
+	cache cacheStore
+	// transaction stuff
+	transactionBuffer []transactionItem
+	transactionMux    sync.Mutex
+	transactionON     = false
+	transactionLock   = false
 )
 
-// The main function to cache with expiration
-func (xe *XEntry) XCache(key string, expire time.Duration, value expiringCacheEntry) {
-	xe.keepAlive = true
-	xe.key = key
-	xe.expireDuration = expire
-	xe.timestamp = time.Now()
-	xMux.Lock()
-	if _, ok := xcache[key]; !ok {
-		// only count if the key is not already there
-		count(key)
-	}
-	xcache[key] = value
-	xMux.Unlock()
-	go xe.expire()
+func BeginTransaction() {
+	transactionMux.Lock()
+	transactionLock = true
+	transactionON = true
 }
 
-// The internal mechanism for expiartion
-func (xe *XEntry) expire() {
-	for xe.keepAlive {
-		xe.Lock()
-		xe.keepAlive = false
-		xe.Unlock()
-		xe.t = time.NewTimer(xe.expireDuration)
-		<-xe.t.C
-		if !xe.keepAlive {
-			xMux.Lock()
-			if _, ok := xcache[xe.key]; ok {
-				delete(xcache, xe.key)
-				descount(xe.key)
-			}
-			xMux.Unlock()
+func RollbackTransaction() {
+	transactionBuffer = nil
+	transactionLock = false
+	transactionON = false
+	transactionMux.Unlock()
+}
+
+func CommitTransaction() {
+	transactionON = false
+	// apply all transactioned items
+	mux.Lock()
+	for _, item := range transactionBuffer {
+		switch item.kind {
+		case KIND_REM:
+			RemKey(item.key)
+		case KIND_PRF:
+			RemPrefixKey(item.key)
+		case KIND_ADD:
+			Cache(item.key, item.value)
+		case KIND_ADP:
+			CachePush(item.key, item.value)
 		}
 	}
-}
-
-// Getter for the timer
-func (xe *XEntry) timer() *time.Timer {
-	return xe.t
-}
-
-func (xe *XEntry) age() time.Duration {
-	return time.Since(xe.timestamp)
-
-}
-
-// Mark entry to be kept another expirationDuration period
-func (xe *XEntry) KeepAlive() {
-	xe.Lock()
-	defer xe.Unlock()
-	xe.keepAlive = true
-}
-
-// Get an entry from the expiration cache and mark it for keeping alive
-func GetXCached(key string) (ece expiringCacheEntry, err error) {
-	xMux.RLock()
-	defer xMux.RUnlock()
-	if r, ok := xcache[key]; ok {
-		r.KeepAlive()
-		return r, nil
-	}
-	return nil, errors.New("not found")
+	mux.Unlock()
+	transactionBuffer = nil
+	transactionLock = false
+	transactionMux.Unlock()
 }
 
 // The function to be used to cache a key/value pair when expiration is not needed
 func Cache(key string, value interface{}) {
-	mux.Lock()
-	defer mux.Unlock()
-	if _, ok := cache[key]; !ok {
-		// only count if the key is not already there
-		count(key)
+	if !transactionLock {
+		mux.Lock()
+		defer mux.Unlock()
+	}
+	if !transactionON {
+		cache.Put(key, value)
+		//fmt.Println("ADD: ", key)
+	} else {
+		transactionBuffer = append(transactionBuffer, transactionItem{key: key, value: value, kind: KIND_ADD})
+	}
+}
+
+// Appends to an existing slice in the cache key
+func CachePush(key string, value interface{}) {
+	if !transactionLock {
+		mux.Lock()
+		defer mux.Unlock()
+	}
+	if !transactionON {
+		cache.Append(key, value)
+	} else {
+		transactionBuffer = append(transactionBuffer, transactionItem{key: key, value: value, kind: KIND_ADP})
 	}
-	cache[key] = timestampedValue{time.Now(), value}
 }
 
 // The function to extract a value for a key that never expire
 func GetCached(key string) (v interface{}, err error) {
 	mux.RLock()
 	defer mux.RUnlock()
-	if r, ok := cache[key]; ok {
-		return r.value, nil
-	}
-	return nil, errors.New("not found")
+	return cache.Get(key)
 }
 
 func GetKeyAge(key string) (time.Duration, error) {
 	mux.RLock()
 	defer mux.RUnlock()
-	if r, ok := cache[key]; ok {
-		return time.Since(r.timestamp), nil
-	}
-	xMux.RLock()
-	defer xMux.RUnlock()
-	if r, ok := xcache[key]; ok {
-		return r.age(), nil
-	}
-	return 0, errors.New("not found")
+	return cache.GetAge(key)
 }
 
 func RemKey(key string) {
-	mux.Lock()
-	if _, ok := cache[key]; ok {
-		delete(cache, key)
-		descount(key)
+	if !transactionLock {
+		mux.Lock()
+		defer mux.Unlock()
 	}
-	mux.Unlock()
-	xMux.Lock()
-	if r, ok := xcache[key]; ok {
-		if r.timer() != nil {
-			r.timer().Stop()
-		}
+	if !transactionON {
+		cache.Delete(key)
+	} else {
+		transactionBuffer = append(transactionBuffer, transactionItem{key: key, kind: KIND_REM})
 	}
-	if _, ok := xcache[key]; ok {
-		delete(xcache, key)
-		descount(key)
-	}
-	xMux.Unlock()
 }
 
 func RemPrefixKey(prefix string) {
-	mux.Lock()
-	for key, _ := range cache {
-		if strings.HasPrefix(key, prefix) {
-			delete(cache, key)
-			descount(key)
-		}
+	if !transactionLock {
+		mux.Lock()
+		defer mux.Unlock()
 	}
-	mux.Unlock()
-	xMux.Lock()
-	for key, _ := range xcache {
-		if strings.HasPrefix(key, prefix) {
-			if r, ok := xcache[key]; ok {
-				if r.timer() != nil {
-					r.timer().Stop()
-				}
-			}
-			delete(xcache, key)
-			descount(key)
-		}
+	if !transactionON {
+		cache.DeletePrefix(prefix)
+	} else {
+		transactionBuffer = append(transactionBuffer, transactionItem{key: prefix, kind: KIND_PRF})
 	}
-	xMux.Unlock()
-}
-
-func GetAllEntries(prefix string) map[string]interface{} {
-	mux.Lock()
-	result := make(map[string]interface{})
-	for key, timestampedValue := range cache {
-		if strings.HasPrefix(key, prefix) {
-			result[key] = timestampedValue.value
-		}
-	}
-	mux.Unlock()
-	xMux.Lock()
-	for key, value := range xcache {
-		if strings.HasPrefix(key, prefix) {
-			result[key] = value
-		}
-	}
-	xMux.Unlock()
-	return result
 }
 
 // Delete all keys from cache
 func Flush() {
 	mux.Lock()
-	cache = make(map[string]timestampedValue)
-	mux.Unlock()
-	xMux.Lock()
-	for _, v := range xcache {
-		if v.timer() != nil {
-			v.timer().Stop()
-		}
-	}
-	xcache = make(map[string]expiringCacheEntry)
-	xMux.Unlock()
-	cMux.Lock()
-	counters = make(map[string]int64)
-	cMux.Unlock()
-}
-
-func CountEntries(prefix string) (result int64) {
-	if _, ok := counters[prefix]; ok {
-		return counters[prefix]
-	}
-	return 0
-}
-
-// increments the counter for the specified key prefix
-func count(key string) {
-	if len(key) < PREFIX_LEN {
-		return
-	}
-	cMux.Lock()
-	defer cMux.Unlock()
-	prefix := key[:PREFIX_LEN]
-	if _, ok := counters[prefix]; ok {
-		// increase the value
-		counters[prefix] += 1
+	defer mux.Unlock()
+	if DOUBLE_CACHE {
+		cache = newDoubleStore()
 	} else {
-		counters[prefix] = 1
+		cache = newSimpleStore()
 	}
 }
 
-// decrements the counter for the specified key prefix
-func descount(key string) {
-	if len(key) < PREFIX_LEN {
-		return
-	}
-	cMux.Lock()
-	defer cMux.Unlock()
-	prefix := key[:PREFIX_LEN]
-	if value, ok := counters[prefix]; ok && value > 0 {
-		counters[prefix] -= 1
-	}
+func CountEntries(prefix string) (result int) {
+	mux.RLock()
+	defer mux.RUnlock()
+	return cache.CountEntriesForPrefix(prefix)
+}
+
+func GetAllEntries(prefix string) (map[string]timestampedValue, error) {
+	mux.RLock()
+	defer mux.RUnlock()
+	return cache.GetAllForPrefix(prefix)
 }
 
 func GetEntriesKeys(prefix string) (keys []string) {
 	mux.RLock()
 	defer mux.RUnlock()
-	for key, _ := range cache {
-		if strings.HasPrefix(key, prefix) {
-			keys = append(keys, key)
-		}
-	}
-	return
-}
-
-func XGetEntriesKeys(prefix string) (keys []string) {
-	xMux.RLock()
-	defer xMux.RUnlock()
-	for key, _ := range xcache {
-		if strings.HasPrefix(key, prefix) {
-			keys = append(keys, key)
-		}
-	}
-	return
+	return cache.GetKeysForPrefix(prefix)
 }

cache2go/cache_test.go

@@ -1,83 +1,11 @@
 package cache2go
 
-import (
-	"testing"
-	"time"
-)
-
-type myStruct struct {
-	XEntry
-	data string
-}
-
-func TestCache(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 1*time.Second, a)
-	b, err := GetXCached("mama")
-	if err != nil || b == nil || b != a {
-		t.Error("Error retriving data from cache", err)
-	}
-}
-
-func TestCacheExpire(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 1*time.Second, a)
-	b, err := GetXCached("mama")
-	if err != nil || b == nil || b.(*myStruct).data != "mama are mere" {
-		t.Error("Error retriving data from cache", err)
-	}
-	time.Sleep(1001 * time.Millisecond)
-	b, err = GetXCached("mama")
-	if err == nil || b != nil {
-		t.Error("Error expiring data")
-	}
-}
-
-func TestCacheKeepAlive(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 1*time.Second, a)
-	b, err := GetXCached("mama")
-	if err != nil || b == nil || b.(*myStruct).data != "mama are mere" {
-		t.Error("Error retriving data from cache", err)
-	}
-	time.Sleep(500 * time.Millisecond)
-	b.KeepAlive()
-	time.Sleep(501 * time.Millisecond)
-	if err != nil {
-		t.Error("Error keeping cached data alive", err)
-	}
-	time.Sleep(1000 * time.Millisecond)
-	b, err = GetXCached("mama")
-	if err == nil || b != nil {
-		t.Error("Error expiring data")
-	}
-}
-
-func TestFlush(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 10*time.Second, a)
-	time.Sleep(1000 * time.Millisecond)
-	Flush()
-	b, err := GetXCached("mama")
-	if err == nil || b != nil {
-		t.Error("Error expiring data")
-	}
-}
-
-func TestFlushNoTimout(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 10*time.Second, a)
-	Flush()
-	b, err := GetXCached("mama")
-	if err == nil || b != nil {
-		t.Error("Error expiring data")
-	}
-}
+import "testing"
 
 func TestRemKey(t *testing.T) {
 	Cache("t11_mm", "test")
 	if t1, err := GetCached("t11_mm"); err != nil || t1 != "test" {
-		t.Error("Error setting cache")
+		t.Error("Error setting cache: ", err, t1)
 	}
 	RemKey("t11_mm")
 	if t1, err := GetCached("t11_mm"); err == nil || t1 == "test" {
@@ -85,60 +13,84 @@ func TestRemKey(t *testing.T) {
 	}
 }
 
-func TestXRemKey(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 10*time.Second, a)
-	if t1, err := GetXCached("mama"); err != nil || t1 != a {
-		t.Error("Error setting xcache")
+func TestTransaction(t *testing.T) {
+	BeginTransaction()
+	Cache("t11_mm", "test")
+	if t1, err := GetCached("t11_mm"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
 	}
-	RemKey("mama")
-	if t1, err := GetXCached("mama"); err == nil || t1 == a {
-		t.Error("Error removing xcached key: ", err, t1)
+	Cache("t12_mm", "test")
+	RemKey("t11_mm")
+	CommitTransaction()
+	if t1, err := GetCached("t12_mm"); err != nil || t1 != "test" {
+		t.Error("Error commiting transaction")
+	}
+	if t1, err := GetCached("t11_mm"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
 	}
 }
 
-/*
-These tests sometimes fails on drone.io
-func TestGetKeyAge(t *testing.T) {
-	Cache("t1", "test")
-	d, err := GetKeyAge("t1")
-	if err != nil || d > time.Millisecond || d < time.Nanosecond {
-		t.Error("Error getting cache key age: ", d)
+func TestTransactionRem(t *testing.T) {
+	BeginTransaction()
+	Cache("t21_mm", "test")
+	Cache("t21_nn", "test")
+	RemPrefixKey("t21_")
+	CommitTransaction()
+	if t1, err := GetCached("t21_mm"); err == nil || t1 == "test" {
+		t.Error("Error commiting transaction")
+	}
+	if t1, err := GetCached("t21_nn"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
 	}
 }
 
-
-func TestXGetKeyAge(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("t1", 10*time.Second, a)
-	d, err := GetXKeyAge("t1")
-	if err != nil || d > time.Millisecond || d < time.Nanosecond {
-		t.Error("Error getting cache key age: ", d)
+func TestTransactionRollback(t *testing.T) {
+	BeginTransaction()
+	Cache("t31_mm", "test")
+	if t1, err := GetCached("t31_mm"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
+	}
+	Cache("t32_mm", "test")
+	RollbackTransaction()
+	if t1, err := GetCached("t32_mm"); err == nil || t1 == "test" {
+		t.Error("Error commiting transaction")
+	}
+	if t1, err := GetCached("t31_mm"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
+	}
+}
+
+func TestTransactionRemBefore(t *testing.T) {
+	BeginTransaction()
+	RemPrefixKey("t41_")
+	Cache("t41_mm", "test")
+	Cache("t41_nn", "test")
+	CommitTransaction()
+	if t1, err := GetCached("t41_mm"); err != nil || t1 != "test" {
+		t.Error("Error commiting transaction")
+	}
+	if t1, err := GetCached("t41_nn"); err != nil || t1 != "test" {
+		t.Error("Error in transaction cache")
 	}
 }
-*/
 
 func TestRemPrefixKey(t *testing.T) {
-	Cache("x_t1", "test")
-	Cache("y_t1", "test")
-	RemPrefixKey("x_")
-	_, errX := GetCached("x_t1")
-	_, errY := GetCached("y_t1")
+	Cache("xxx_t1", "test")
+	Cache("yyy_t1", "test")
+	RemPrefixKey("xxx_")
+	_, errX := GetCached("xxx_t1")
+	_, errY := GetCached("yyy_t1")
 	if errX == nil || errY != nil {
 		t.Error("Error removing prefix: ", errX, errY)
 	}
 }
 
-func TestXRemPrefixKey(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("x_t1", 10*time.Second, a)
-	a.XCache("y_t1", 10*time.Second, a)
-
-	RemPrefixKey("x_")
-	_, errX := GetXCached("x_t1")
-	_, errY := GetXCached("y_t1")
-	if errX == nil || errY != nil {
-		t.Error("Error removing prefix: ", errX, errY)
+func TestCachePush(t *testing.T) {
+	CachePush("ccc_t1", "1")
+	CachePush("ccc_t1", "2")
+	v, err := GetCached("ccc_t1")
+	if err != nil || len(v.(map[interface{}]struct{})) != 2 {
+		t.Error("Error in cache push: ", v)
 	}
 }
 

cache2go/store.go

@@ -0,0 +1,223 @@
+//Simple caching library with expiration capabilities
+package cache2go
+
+import (
+	"strings"
+	"time"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+type cacheStore interface {
+	Put(string, interface{})
+	Append(string, interface{})
+	Get(string) (interface{}, error)
+	GetAge(string) (time.Duration, error)
+	Delete(string)
+	DeletePrefix(string)
+	CountEntriesForPrefix(string) int
+	GetAllForPrefix(string) (map[string]timestampedValue, error)
+	GetKeysForPrefix(string) []string
+}
+
+// easy to be counted exported by prefix
+type cacheDoubleStore map[string]map[string]timestampedValue
+
+func newDoubleStore() cacheDoubleStore {
+	return make(cacheDoubleStore)
+}
+
+func (cs cacheDoubleStore) Put(key string, value interface{}) {
+	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
+	if _, ok := cs[prefix]; !ok {
+		cs[prefix] = make(map[string]timestampedValue)
+	}
+	cs[prefix][key] = timestampedValue{time.Now(), value}
+}
+
+func (cs cacheDoubleStore) Append(key string, value interface{}) {
+	var elements map[interface{}]struct{} // using map for faster check if element is present
+	if v, err := cs.Get(key); err == nil {
+		elements = v.(map[interface{}]struct{})
+	} else {
+		elements = make(map[interface{}]struct{})
+	}
+	elements[value] = struct{}{}
+	cache.Put(key, elements)
+}
+
+func (cs cacheDoubleStore) Get(key string) (interface{}, error) {
+	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
+	if keyMap, ok := cs[prefix]; ok {
+		if ti, exists := keyMap[key]; exists {
+			return ti.value, nil
+		}
+	}
+	return nil, utils.ErrNotFound
+}
+
+func (cs cacheDoubleStore) GetAge(key string) (time.Duration, error) {
+	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
+	if keyMap, ok := cs[prefix]; ok {
+		if ti, exists := keyMap[key]; exists {
+			return time.Since(ti.timestamp), nil
+		}
+	}
+	return -1, utils.ErrNotFound
+}
+
+func (cs cacheDoubleStore) Delete(key string) {
+	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
+	if keyMap, ok := cs[prefix]; ok {
+		delete(keyMap, key)
+	}
+}
+
+func (cs cacheDoubleStore) DeletePrefix(prefix string) {
+	delete(cs, prefix)
+}
+
+func (cs cacheDoubleStore) CountEntriesForPrefix(prefix string) int {
+	if m, ok := cs[prefix]; ok {
+		return len(m)
+	}
+	return 0
+}
+
+func (cs cacheDoubleStore) GetAllForPrefix(prefix string) (map[string]timestampedValue, error) {
+	if keyMap, ok := cs[prefix]; ok {
+		return keyMap, nil
+	}
+	return nil, utils.ErrNotFound
+}
+
+func (cs cacheDoubleStore) GetKeysForPrefix(prefix string) (keys []string) {
+	prefix, key := prefix[:PREFIX_LEN], prefix[PREFIX_LEN:]
+	if keyMap, ok := cs[prefix]; ok {
+		for iterKey := range keyMap {
+			if len(key) > 0 && strings.HasPrefix(iterKey, key) {
+				keys = append(keys, prefix+iterKey)
+			}
+		}
+	}
+	return
+}
+
+// faster to access
+type cacheSimpleStore struct {
+	cache    map[string]timestampedValue
+	counters map[string]int
+}
+
+func newSimpleStore() cacheSimpleStore {
+	return cacheSimpleStore{
+		cache:    make(map[string]timestampedValue),
+		counters: make(map[string]int),
+	}
+}
+
+func (cs cacheSimpleStore) Put(key string, value interface{}) {
+	if _, ok := cs.cache[key]; !ok {
+		// only count if the key is not already there
+		cs.count(key)
+	}
+	cs.cache[key] = timestampedValue{time.Now(), value}
+}
+
+func (cs cacheSimpleStore) Append(key string, value interface{}) {
+	var elements map[interface{}]struct{}
+	if v, err := cs.Get(key); err == nil {
+		elements = v.(map[interface{}]struct{})
+	} else {
+		elements = make(map[interface{}]struct{})
+	}
+	elements[value] = struct{}{}
+	cache.Put(key, elements)
+}
+
+func (cs cacheSimpleStore) Get(key string) (interface{}, error) {
+	if ti, exists := cs.cache[key]; exists {
+		return ti.value, nil
+	}
+	return nil, utils.ErrNotFound
+}
+
+func (cs cacheSimpleStore) GetAge(key string) (time.Duration, error) {
+	if ti, exists := cs.cache[key]; exists {
+		return time.Since(ti.timestamp), nil
+	}
+
+	return -1, utils.ErrNotFound
+}
+
+func (cs cacheSimpleStore) Delete(key string) {
+	if _, ok := cs.cache[key]; ok {
+		delete(cs.cache, key)
+		cs.descount(key)
+	}
+}
+
+func (cs cacheSimpleStore) DeletePrefix(prefix string) {
+	for key, _ := range cs.cache {
+		if strings.HasPrefix(key, prefix) {
+			delete(cs.cache, key)
+			cs.descount(key)
+		}
+	}
+}
+
+// increments the counter for the specified key prefix
+func (cs cacheSimpleStore) count(key string) {
+	if len(key) < PREFIX_LEN {
+		return
+	}
+	prefix := key[:PREFIX_LEN]
+	if _, ok := cs.counters[prefix]; ok {
+		// increase the value
+		cs.counters[prefix] += 1
+	} else {
+		cs.counters[prefix] = 1
+	}
+}
+
+// decrements the counter for the specified key prefix
+func (cs cacheSimpleStore) descount(key string) {
+	if len(key) < PREFIX_LEN {
+		return
+	}
+	prefix := key[:PREFIX_LEN]
+	if value, ok := cs.counters[prefix]; ok && value > 0 {
+		cs.counters[prefix] -= 1
+	}
+}
+
+func (cs cacheSimpleStore) CountEntriesForPrefix(prefix string) int {
+	if _, ok := cs.counters[prefix]; ok {
+		return cs.counters[prefix]
+	}
+	return 0
+}
+
+func (cs cacheSimpleStore) GetAllForPrefix(prefix string) (map[string]timestampedValue, error) {
+	result := make(map[string]timestampedValue)
+	found := false
+	for key, ti := range cs.cache {
+		if strings.HasPrefix(key, prefix) {
+			result[key[PREFIX_LEN:]] = ti
+			found = true
+		}
+	}
+	if !found {
+		return nil, utils.ErrNotFound
+	}
+	return result, nil
+}
+
+func (cs cacheSimpleStore) GetKeysForPrefix(prefix string) (keys []string) {
+	for key, _ := range cs.cache {
+		if strings.HasPrefix(key, prefix) {
+			keys = append(keys, key)
+		}
+	}
+	return
+}

calls_test.sh

@@ -0,0 +1,9 @@
+#! /usr/bin/env sh
+
+./local_test.sh
+lcl=$?
+echo 'go test github.com/cgrates/cgrates/general_tests -calls'
+go test github.com/cgrates/cgrates/general_tests -calls
+gnr=$?
+
+exit $gen && $gnr

cdrc/cdrc.go

@@ -1,6 +1,6 @@
 /*
 Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
+Copyright (C) ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -27,28 +27,112 @@ import (
 	"net/http"
 	"os"
 	"path"
-	"strconv"
-	"strings"
 	"time"
-	"unicode/utf8"
 
+	"github.com/cgrates/cgrates/config"
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
-	"github.com/howeyc/fsnotify"
+	"gopkg.in/fsnotify.v1"
 )
 
 const (
-	CSV    = "csv"
-	FS_CSV = "freeswitch_csv"
+	CSV             = "csv"
+	FS_CSV          = "freeswitch_csv"
+	UNPAIRED_SUFFIX = ".unpaired"
 )
 
-func NewCdrc(cdrsAddress, cdrType, cdrInDir, cdrOutDir, cdrSourceId string, runDelay time.Duration, csvSep string, cdrFields map[string][]*utils.RSRField, cdrServer *engine.CDRS) (*Cdrc, error) {
-	if len(csvSep) != 1 {
-		return nil, fmt.Errorf("Unsupported csv separator: %s", csvSep)
+// Populates the
+func populateStoredCdrField(cdr *engine.StoredCdr, fieldId, fieldVal string) error {
+	var err error
+	switch fieldId {
+	case utils.TOR:
+		cdr.TOR += fieldVal
+	case utils.ACCID:
+		cdr.AccId += fieldVal
+	case utils.REQTYPE:
+		cdr.ReqType += fieldVal
+	case utils.DIRECTION:
+		cdr.Direction += fieldVal
+	case utils.TENANT:
+		cdr.Tenant += fieldVal
+	case utils.CATEGORY:
+		cdr.Category += fieldVal
+	case utils.ACCOUNT:
+		cdr.Account += fieldVal
+	case utils.SUBJECT:
+		cdr.Subject += fieldVal
+	case utils.DESTINATION:
+		cdr.Destination += fieldVal
+	case utils.SETUP_TIME:
+		if cdr.SetupTime, err = utils.ParseTimeDetectLayout(fieldVal); err != nil {
+			return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
+		}
+	case utils.PDD:
+		if cdr.Pdd, err = utils.ParseDurationWithSecs(fieldVal); err != nil {
+			return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
+		}
+	case utils.ANSWER_TIME:
+		if cdr.AnswerTime, err = utils.ParseTimeDetectLayout(fieldVal); err != nil {
+			return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
+		}
+	case utils.USAGE:
+		if cdr.Usage, err = utils.ParseDurationWithSecs(fieldVal); err != nil {
+			return fmt.Errorf("Cannot parse duration field with value: %s, err: %s", fieldVal, err.Error())
+		}
+	case utils.SUPPLIER:
+		cdr.Supplier += fieldVal
+	case utils.DISCONNECT_CAUSE:
+		cdr.DisconnectCause += fieldVal
+	default: // Extra fields will not match predefined so they all show up here
+		cdr.ExtraFields[fieldId] += fieldVal
+	}
+	return nil
+}
+
+// Understands and processes a specific format of cdr (eg: .csv or .fwv)
+type RecordsProcessor interface {
+	ProcessNextRecord() ([]*engine.StoredCdr, error) // Process a single record in the CDR file, return a slice of CDRs since based on configuration we can have more templates
+}
+
+/*
+One instance  of CDRC will act on one folder.
+Common parameters within configs processed:
+ * cdrS, cdrFormat, cdrInDir, cdrOutDir, runDelay
+Parameters specific per config instance:
+ * duMultiplyFactor, cdrSourceId, cdrFilter, cdrFields
+*/
+func NewCdrc(cdrcCfgs map[string]*config.CdrcConfig, httpSkipTlsCheck bool, cdrs engine.Connector, exitChan chan struct{}) (*Cdrc, error) {
+	var cdrcCfg *config.CdrcConfig
+	for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one
+		break
+	}
+	cdrc := &Cdrc{cdrFormat: cdrcCfg.CdrFormat, cdrInDir: cdrcCfg.CdrInDir, cdrOutDir: cdrcCfg.CdrOutDir,
+		runDelay: cdrcCfg.RunDelay, csvSep: cdrcCfg.FieldSeparator,
+		httpSkipTlsCheck: httpSkipTlsCheck, cdrcCfgs: cdrcCfgs, dfltCdrcCfg: cdrcCfg, cdrs: cdrs, exitChan: exitChan, maxOpenFiles: make(chan struct{}, cdrcCfg.MaxOpenFiles),
+	}
+	var processFile struct{}
+	for i := 0; i < cdrcCfg.MaxOpenFiles; i++ {
+		cdrc.maxOpenFiles <- processFile // Empty initiate so we do not need to wait later when we pop
+	}
+	cdrc.cdrSourceIds = make([]string, len(cdrcCfgs))
+	cdrc.duMultiplyFactors = make([]float64, len(cdrcCfgs))
+	cdrc.cdrFilters = make([]utils.RSRFields, len(cdrcCfgs))
+	cdrc.cdrFields = make([][]*config.CfgCdrField, len(cdrcCfgs))
+	idx := 0
+	var err error
+	for _, cfg := range cdrcCfgs {
+		if idx == 0 { // Steal the config from just one instance since it should be the same for all
+			cdrc.failedCallsPrefix = cfg.FailedCallsPrefix
+			if cdrc.partialRecordsCache, err = NewPartialRecordsCache(cdrcCfg.PartialRecordCache, cdrcCfg.CdrOutDir, cdrcCfg.FieldSeparator); err != nil {
+				return nil, err
+			}
+		}
+		cdrc.cdrSourceIds[idx] = cfg.CdrSourceId
+		cdrc.duMultiplyFactors[idx] = cfg.DataUsageMultiplyFactor
+		cdrc.cdrFilters[idx] = cfg.CdrFilter
+		cdrc.cdrFields[idx] = cfg.ContentFields
+		idx += 1
 	}
-	csvSepRune, _ := utf8.DecodeRune([]byte(csvSep))
-	cdrc := &Cdrc{cdrsAddress: cdrsAddress, cdrType: cdrType, cdrInDir: cdrInDir, cdrOutDir: cdrOutDir,
-		cdrSourceId: cdrSourceId, runDelay: runDelay, csvSep: csvSepRune, cdrFields: cdrFields, cdrServer: cdrServer}
 	// Before processing, make sure in and out folders exist
 	for _, dir := range []string{cdrc.cdrInDir, cdrc.cdrOutDir} {
 		if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
@@ -60,16 +144,24 @@ func NewCdrc(cdrsAddress, cdrType, cdrInDir, cdrOutDir, cdrSourceId string, runD
 }
 
 type Cdrc struct {
-	cdrsAddress,
-	cdrType,
+	cdrFormat,
 	cdrInDir,
-	cdrOutDir,
-	cdrSourceId string
-	runDelay   time.Duration
-	csvSep     rune
-	cdrFields  map[string][]*utils.RSRField
-	cdrServer  *engine.CDRS // Reference towards internal cdrServer if that is the case
-	httpClient *http.Client
+	cdrOutDir string
+	failedCallsPrefix   string   // Configured failedCallsPrefix, used in case of flatstore CDRs
+	cdrSourceIds        []string // Should be in sync with cdrFields on indexes
+	runDelay            time.Duration
+	csvSep              rune
+	duMultiplyFactors   []float64
+	cdrFilters          []utils.RSRFields       // Should be in sync with cdrFields on indexes
+	cdrFields           [][]*config.CfgCdrField // Profiles directly connected with cdrFilters
+	httpSkipTlsCheck    bool
+	cdrcCfgs            map[string]*config.CdrcConfig // All cdrc config profiles attached to this CDRC (key will be profile instance name)
+	dfltCdrcCfg         *config.CdrcConfig
+	cdrs                engine.Connector
+	httpClient          *http.Client
+	exitChan            chan struct{}
+	maxOpenFiles        chan struct{}        // Maximum number of simultaneous files processed
+	partialRecordsCache *PartialRecordsCache // Shared between all files in the folder we process
 }
 
 // When called fires up folder monitoring, either automated via inotify or manual by sleeping between processing
@@ -77,72 +169,50 @@ func (self *Cdrc) Run() error {
 	if self.runDelay == time.Duration(0) { // Automated via inotify
 		return self.trackCDRFiles()
 	}
-	// No automated, process and sleep approach
+	// Not automated, process and sleep approach
 	for {
+		select {
+		case exitChan := <-self.exitChan: // Exit, reinject exitChan for other CDRCs
+			self.exitChan <- exitChan
+			engine.Logger.Info(fmt.Sprintf("<Cdrc> Shutting down CDRC on path %s.", self.cdrInDir))
+			return nil
+		default:
+		}
 		self.processCdrDir()
 		time.Sleep(self.runDelay)
 	}
 }
 
-// Takes the record out of csv and turns it into http form which can be posted
-func (self *Cdrc) recordToStoredCdr(record []string) (*utils.StoredCdr, error) {
-	storedCdr := &utils.StoredCdr{CdrHost: "0.0.0.0", CdrSource: self.cdrSourceId, ExtraFields: make(map[string]string), Cost: -1}
-	var err error
-	for cfgFieldName, cfgFieldRSRs := range self.cdrFields {
-		var fieldVal string
-		if utils.IsSliceMember([]string{CSV, FS_CSV}, self.cdrType) {
-			for _, cfgFieldRSR := range cfgFieldRSRs {
-				if strings.HasPrefix(cfgFieldRSR.Id, utils.STATIC_VALUE_PREFIX) {
-					fieldVal += cfgFieldRSR.ParseValue("PLACEHOLDER")
-				} else { // Dynamic value extracted using index
-					if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx {
-						return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cfgFieldName)
-					} else {
-						fieldVal += cfgFieldRSR.ParseValue(record[cfgFieldIdx])
-					}
-				}
-			}
-		} else { // Modify here when we add more supported cdr formats
-			fieldVal = "UNKNOWN"
-		}
-		switch cfgFieldName {
-		case utils.TOR:
-			storedCdr.TOR = fieldVal
-		case utils.ACCID:
-			storedCdr.AccId = fieldVal
-		case utils.REQTYPE:
-			storedCdr.ReqType = fieldVal
-		case utils.DIRECTION:
-			storedCdr.Direction = fieldVal
-		case utils.TENANT:
-			storedCdr.Tenant = fieldVal
-		case utils.CATEGORY:
-			storedCdr.Category = fieldVal
-		case utils.ACCOUNT:
-			storedCdr.Account = fieldVal
-		case utils.SUBJECT:
-			storedCdr.Subject = fieldVal
-		case utils.DESTINATION:
-			storedCdr.Destination = fieldVal
-		case utils.SETUP_TIME:
-			if storedCdr.SetupTime, err = utils.ParseTimeDetectLayout(fieldVal); err != nil {
-				return nil, fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
-			}
-		case utils.ANSWER_TIME:
-			if storedCdr.AnswerTime, err = utils.ParseTimeDetectLayout(fieldVal); err != nil {
-				return nil, fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
-			}
-		case utils.USAGE:
-			if storedCdr.Usage, err = utils.ParseDurationWithNanosecs(fieldVal); err != nil {
-				return nil, fmt.Errorf("Cannot parse duration field with value: %s, err: %s", fieldVal, err.Error())
-			}
-		default: // Extra fields will not match predefined so they all show up here
-			storedCdr.ExtraFields[cfgFieldName] = fieldVal
-		}
-
+// Watch the specified folder for file moves and parse the files on events
+func (self *Cdrc) trackCDRFiles() (err error) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return
+	}
+	defer watcher.Close()
+	err = watcher.Add(self.cdrInDir)
+	if err != nil {
+		return
+	}
+	engine.Logger.Info(fmt.Sprintf("<Cdrc> Monitoring %s for file moves.", self.cdrInDir))
+	for {
+		select {
+		case exitChan := <-self.exitChan: // Exit, reinject exitChan for other CDRCs
+			self.exitChan <- exitChan
+			engine.Logger.Info(fmt.Sprintf("<Cdrc> Shutting down CDRC on path %s.", self.cdrInDir))
+			return nil
+		case ev := <-watcher.Events:
+			if ev.Op&fsnotify.Create == fsnotify.Create && (self.cdrFormat != FS_CSV || path.Ext(ev.Name) != ".csv") {
+				go func() { //Enable async processing here
+					if err = self.processFile(ev.Name); err != nil {
+						engine.Logger.Err(fmt.Sprintf("Processing file %s, error: %s", ev.Name, err.Error()))
+					}
+				}()
+			}
+		case err := <-watcher.Errors:
+			engine.Logger.Err(fmt.Sprintf("Inotify error: %s", err.Error()))
+		}
 	}
-	storedCdr.CgrId = utils.Sha1(storedCdr.AccId, storedCdr.SetupTime.String())
-	return storedCdr, nil
 }
 
 // One run over the CDR folder
@@ -150,7 +220,7 @@ func (self *Cdrc) processCdrDir() error {
 	engine.Logger.Info(fmt.Sprintf("<Cdrc> Parsing folder %s for CDR files.", self.cdrInDir))
 	filesInDir, _ := ioutil.ReadDir(self.cdrInDir)
 	for _, file := range filesInDir {
-		if self.cdrType != FS_CSV || path.Ext(file.Name()) != ".csv" {
+		if self.cdrFormat != FS_CSV || path.Ext(file.Name()) != ".csv" {
 			go func() { //Enable async processing here
 				if err := self.processFile(path.Join(self.cdrInDir, file.Name())); err != nil {
 					engine.Logger.Err(fmt.Sprintf("Processing file %s, error: %s", file, err.Error()))
@@ -161,36 +231,12 @@ func (self *Cdrc) processCdrDir() error {
 	return nil
 }
 
-// Watch the specified folder for file moves and parse the files on events
-func (self *Cdrc) trackCDRFiles() (err error) {
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		return
-	}
-	defer watcher.Close()
-	err = watcher.Watch(self.cdrInDir)
-	if err != nil {
-		return
-	}
-	engine.Logger.Info(fmt.Sprintf("<Cdrc> Monitoring %s for file moves.", self.cdrInDir))
-	for {
-		select {
-		case ev := <-watcher.Event:
-			if ev.IsCreate() && (self.cdrType != FS_CSV || path.Ext(ev.Name) != ".csv") {
-				go func() { //Enable async processing here
-					if err = self.processFile(ev.Name); err != nil {
-						engine.Logger.Err(fmt.Sprintf("Processing file %s, error: %s", ev.Name, err.Error()))
-					}
-				}()
-			}
-		case err := <-watcher.Error:
-			engine.Logger.Err(fmt.Sprintf("Inotify error: %s", err.Error()))
-		}
-	}
-}
-
 // Processe file at filePath and posts the valid cdr rows out of it
 func (self *Cdrc) processFile(filePath string) error {
+	if cap(self.maxOpenFiles) != 0 { // 0 goes for no limit
+		processFile := <-self.maxOpenFiles // Queue here for maxOpenFiles
+		defer func() { self.maxOpenFiles <- processFile }()
+	}
 	_, fn := path.Split(filePath)
 	engine.Logger.Info(fmt.Sprintf("<Cdrc> Parsing: %s", filePath))
 	file, err := os.Open(filePath)
@@ -199,34 +245,43 @@ func (self *Cdrc) processFile(filePath string) error {
 		engine.Logger.Crit(err.Error())
 		return err
 	}
-	csvReader := csv.NewReader(bufio.NewReader(file))
-	csvReader.Comma = self.csvSep
+	var recordsProcessor RecordsProcessor
+	switch self.cdrFormat {
+	case CSV, FS_CSV, utils.KAM_FLATSTORE, utils.OSIPS_FLATSTORE:
+		csvReader := csv.NewReader(bufio.NewReader(file))
+		csvReader.Comma = self.csvSep
+		recordsProcessor = NewCsvRecordsProcessor(csvReader, self.cdrFormat, fn, self.failedCallsPrefix,
+			self.cdrSourceIds, self.duMultiplyFactors, self.cdrFilters, self.cdrFields, self.httpSkipTlsCheck, self.partialRecordsCache)
+	case utils.FWV:
+		recordsProcessor = NewFwvRecordsProcessor(file, self.cdrcCfgs, self.dfltCdrcCfg, self.httpClient, self.httpSkipTlsCheck)
+	default:
+		return fmt.Errorf("Unsupported CDR format: %s", self.cdrFormat)
+	}
 	procRowNr := 0
+	cdrsPosted := 0
 	timeStart := time.Now()
 	for {
-		record, err := csvReader.Read()
+		cdrs, err := recordsProcessor.ProcessNextRecord()
 		if err != nil && err == io.EOF {
-			break // End of file
+			break
 		}
-		procRowNr += 1 // Only increase if not end of file
+		procRowNr += 1
 		if err != nil {
-			engine.Logger.Err(fmt.Sprintf("<Cdrc> Error in csv file: %s", err.Error()))
-			continue // Other csv related errors, ignore
-		}
-		storedCdr, err := self.recordToStoredCdr(record)
-		if err != nil {
-			engine.Logger.Err(fmt.Sprintf("<Cdrc> Error in csv file: %s", err.Error()))
+			engine.Logger.Err(fmt.Sprintf("<Cdrc> Row %d, error: %s", procRowNr, err.Error()))
 			continue
 		}
-		if self.cdrsAddress == utils.INTERNAL {
-			if err := self.cdrServer.ProcessCdr(storedCdr); err != nil {
-				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed posting CDR, row: %d, error: %s", procRowNr, err.Error()))
+		for _, storedCdr := range cdrs { // Send CDRs to CDRS
+			var reply string
+			if self.dfltCdrcCfg.DryRun {
+				engine.Logger.Info(fmt.Sprintf("<Cdrc> DryRun CDR: %+v", storedCdr))
 				continue
 			}
-		} else { // CDRs listening on IP
-			if _, err := self.httpClient.PostForm(fmt.Sprintf("http://%s/cgr", self.cdrsAddress), storedCdr.AsHttpForm()); err != nil {
-				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed posting CDR, row: %d, error: %s", procRowNr, err.Error()))
-				continue
+			if err := self.cdrs.ProcessCdr(storedCdr, &reply); err != nil {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed sending CDR, %+v, error: %s", storedCdr, err.Error()))
+			} else if reply != "OK" {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Received unexpected reply for CDR, %+v, reply: %s", storedCdr, reply))
+			} else {
+				cdrsPosted += 1
 			}
 		}
 	}
@@ -236,7 +291,7 @@ func (self *Cdrc) processFile(filePath string) error {
 		engine.Logger.Err(err.Error())
 		return err
 	}
-	engine.Logger.Info(fmt.Sprintf("Finished processing %s, moved to %s. Total records processed: %d, run duration: %s",
-		fn, newPath, procRowNr, time.Now().Sub(timeStart)))
+	engine.Logger.Info(fmt.Sprintf("Finished processing %s, moved to %s. Total records processed: %d, CDRs posted: %d, run duration: %s",
+		fn, newPath, procRowNr, cdrsPosted, time.Now().Sub(timeStart)))
 	return nil
 }

cdrc/cdrc_local_test.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -22,15 +22,16 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
 	"testing"
 	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
 )
 
 /*
@@ -48,17 +49,14 @@ README:
 
 var cfgPath string
 var cfg *config.CGRConfig
+var cdrcCfgs map[string]*config.CdrcConfig
+var cdrcCfg *config.CdrcConfig
 
 var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args
 var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here")
 var storDbType = flag.String("stordb_type", "mysql", "The type of the storDb database <mysql>")
 var waitRater = flag.Int("wait_rater", 300, "Number of miliseconds to wait for rater to start and cache")
 
-func init() {
-	cfgPath = path.Join(*dataDir, "conf", "samples", "apier_local_test.cfg")
-	cfg, _ = config.NewCGRConfigFromFile(&cfgPath)
-}
-
 var fileContent1 = `accid11,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1
 accid12,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1
 dummy_data
@@ -94,6 +92,18 @@ func stopEngine() error {
 	return nil
 }
 
+// Need it here and not in init since Travis has no possibility to load local file
+func TestLoadConfigt(*testing.T) {
+	if !*testLocal {
+		return
+	}
+	cfgPath = path.Join(*dataDir, "conf", "samples", "apier")
+	cfg, _ = config.NewCGRConfigFromFolder(cfgPath)
+	if len(cfg.CdrcProfiles) > 0 {
+		cdrcCfgs = cfg.CdrcProfiles["/var/log/cgrates/cdrc/in"]
+	}
+}
+
 func TestEmptyTables(t *testing.T) {
 	if !*testLocal {
 		return
@@ -102,12 +112,12 @@ func TestEmptyTables(t *testing.T) {
 		t.Fatal("Unsupported storDbType")
 	}
 	var mysql *engine.MySQLStorage
-	if d, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass); err != nil {
+	if d, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns); err != nil {
 		t.Fatal("Error on opening database connection: ", err)
 	} else {
 		mysql = d.(*engine.MySQLStorage)
 	}
-	for _, scriptName := range []string{engine.CREATE_CDRS_TABLES_SQL, engine.CREATE_TARIFFPLAN_TABLES_SQL} {
+	for _, scriptName := range []string{utils.CREATE_CDRS_TABLES_SQL, utils.CREATE_TARIFFPLAN_TABLES_SQL} {
 		if err := mysql.CreateTablesFromScript(path.Join(*dataDir, "storage", *storDbType, scriptName)); err != nil {
 			t.Fatal("Error on mysql creation: ", err.Error())
 			return // No point in going further
@@ -125,38 +135,48 @@ func TestCreateCdrFiles(t *testing.T) {
 	if !*testLocal {
 		return
 	}
-	if err := os.RemoveAll(cfg.CdrcCdrInDir); err != nil {
-		t.Fatal("Error removing folder: ", cfg.CdrcCdrInDir, err)
+	if cdrcCfgs == nil {
+		t.Fatal("Empty default cdrc configuration")
 	}
-	if err := os.MkdirAll(cfg.CdrcCdrInDir, 0755); err != nil {
-		t.Fatal("Error creating folder: ", cfg.CdrcCdrInDir, err)
+	for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one
+		break
 	}
-	if err := os.RemoveAll(cfg.CdrcCdrOutDir); err != nil {
-		t.Fatal("Error removing folder: ", cfg.CdrcCdrInDir, err)
+	if err := os.RemoveAll(cdrcCfg.CdrInDir); err != nil {
+		t.Fatal("Error removing folder: ", cdrcCfg.CdrInDir, err)
 	}
-	if err := os.MkdirAll(cfg.CdrcCdrOutDir, 0755); err != nil {
-		t.Fatal("Error creating folder: ", cfg.CdrcCdrOutDir, err)
+	if err := os.MkdirAll(cdrcCfg.CdrInDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", cdrcCfg.CdrInDir, err)
 	}
-	if err := ioutil.WriteFile(path.Join(cfg.CdrcCdrInDir, "file1.csv"), []byte(fileContent1), 0644); err != nil {
+	if err := os.RemoveAll(cdrcCfg.CdrOutDir); err != nil {
+		t.Fatal("Error removing folder: ", cdrcCfg.CdrOutDir, err)
+	}
+	if err := os.MkdirAll(cdrcCfg.CdrOutDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", cdrcCfg.CdrOutDir, err)
+	}
+	if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file1.csv"), []byte(fileContent1), 0644); err != nil {
 		t.Fatal(err.Error)
 	}
-	if err := ioutil.WriteFile(path.Join(cfg.CdrcCdrInDir, "file2.csv"), []byte(fileContent2), 0644); err != nil {
+	if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file2.csv"), []byte(fileContent2), 0644); err != nil {
 		t.Fatal(err.Error)
 	}
+
 }
 
 func TestProcessCdrDir(t *testing.T) {
 	if !*testLocal {
 		return
 	}
-	if cfg.CdrcCdrs == utils.INTERNAL { // For now we only test over network
-		return
+	var cdrcCfg *config.CdrcConfig
+	for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one
+		break
+	}
+	if cdrcCfg.Cdrs == utils.INTERNAL { // For now we only test over network
+		cdrcCfg.Cdrs = "127.0.0.1:2013"
 	}
 	if err := startEngine(); err != nil {
 		t.Fatal(err.Error())
 	}
-	cdrc, err := NewCdrc(cfg.CdrcCdrs, cfg.CdrcCdrType, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.CdrcSourceId, cfg.CdrcRunDelay, cfg.CdrcCsvSep,
-		cfg.CdrcCdrFields, nil)
+	cdrc, err := NewCdrc(cdrcCfgs, true, nil, make(chan struct{}))
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -171,13 +191,13 @@ func TestCreateCdr3File(t *testing.T) {
 	if !*testLocal {
 		return
 	}
-	if err := os.RemoveAll(cfg.CdrcCdrInDir); err != nil {
-		t.Fatal("Error removing folder: ", cfg.CdrcCdrInDir, err)
+	if err := os.RemoveAll(cdrcCfg.CdrInDir); err != nil {
+		t.Fatal("Error removing folder: ", cdrcCfg.CdrInDir, err)
 	}
-	if err := os.MkdirAll(cfg.CdrcCdrInDir, 0755); err != nil {
-		t.Fatal("Error creating folder: ", cfg.CdrcCdrInDir, err)
+	if err := os.MkdirAll(cdrcCfg.CdrInDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", cdrcCfg.CdrInDir, err)
 	}
-	if err := ioutil.WriteFile(path.Join(cfg.CdrcCdrInDir, "file3.csv"), []byte(fileContent3), 0644); err != nil {
+	if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file3.csv"), []byte(fileContent3), 0644); err != nil {
 		t.Fatal(err.Error)
 	}
 }
@@ -186,14 +206,13 @@ func TestProcessCdr3Dir(t *testing.T) {
 	if !*testLocal {
 		return
 	}
-	if cfg.CdrcCdrs == utils.INTERNAL { // For now we only test over network
-		return
+	if cdrcCfg.Cdrs == utils.INTERNAL { // For now we only test over network
+		cdrcCfg.Cdrs = "127.0.0.1:2013"
 	}
 	if err := startEngine(); err != nil {
 		t.Fatal(err.Error())
 	}
-	cdrc, err := NewCdrc(cfg.CdrcCdrs, cfg.CdrcCdrType, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.CdrcSourceId, cfg.CdrcRunDelay, ";",
-		cfg.CdrcCdrFields, nil)
+	cdrc, err := NewCdrc(cdrcCfgs, true, nil, make(chan struct{}))
 	if err != nil {
 		t.Fatal(err.Error())
 	}

cdrc/cdrc_test.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -18,172 +18,299 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 
 package cdrc
 
-import (
-	//"bytes"
-	//"encoding/csv"
-	//"fmt"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
-	//"io"
-	"reflect"
-	"testing"
-	"time"
-	"unicode/utf8"
-)
-
-func TestRecordForkCdr(t *testing.T) {
-	cgrConfig, _ := config.NewDefaultCGRConfig()
-	cgrConfig.CdrcCdrFields["supplier"] = []*utils.RSRField{&utils.RSRField{Id: "14"}}
-	csvSepRune, _ := utf8.DecodeRune([]byte(cgrConfig.CdrcCsvSep))
-	cdrc := &Cdrc{cgrConfig.CdrcCdrs, cgrConfig.CdrcCdrType, cgrConfig.CdrcCdrInDir, cgrConfig.CdrcCdrOutDir, cgrConfig.CdrcSourceId, cgrConfig.CdrcRunDelay, csvSepRune,
-		cgrConfig.CdrcCdrFields, new(engine.CDRS), nil}
-	cdrRow := []string{"firstField", "secondField"}
-	_, err := cdrc.recordToStoredCdr(cdrRow)
-	if err == nil {
-		t.Error("Failed to corectly detect missing fields from record")
+/*
+func TestNewPartialFlatstoreRecord(t *testing.T) {
+	ePr := &PartialFlatstoreRecord{Method: "INVITE", AccId: "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:02daec40c548625ac", Timestamp: time.Date(2015, 7, 9, 15, 6, 48, 0, time.UTC),
+		Values: []string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454408", "*prepaid", "1001", "1002", "", "3401:2069362475"}}
+	if pr, err := NewPartialFlatstoreRecord(ePr.Values); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(ePr, pr) {
+		t.Errorf("Expecting: %+v, received: %+v", ePr, pr)
 	}
-	cdrRow = []string{"ignored", "ignored", utils.VOICE, "acc1", "prepaid", "*out", "cgrates.org", "call", "1001", "1001", "+4986517174963",
-		"2013-02-03 19:50:00", "2013-02-03 19:54:00", "62000000000", "supplier1", "172.16.1.1"}
-	rtCdr, err := cdrc.recordToStoredCdr(cdrRow)
-	if err != nil {
-		t.Error("Failed to parse CDR in rated cdr", err)
-	}
-	expectedCdr := &utils.StoredCdr{
-		CgrId:       utils.Sha1(cdrRow[3], time.Date(2013, 2, 3, 19, 50, 0, 0, time.UTC).String()),
-		TOR:         cdrRow[2],
-		AccId:       cdrRow[3],
-		CdrHost:     "0.0.0.0", // Got it over internal interface
-		CdrSource:   cgrConfig.CdrcSourceId,
-		ReqType:     cdrRow[4],
-		Direction:   cdrRow[5],
-		Tenant:      cdrRow[6],
-		Category:    cdrRow[7],
-		Account:     cdrRow[8],
-		Subject:     cdrRow[9],
-		Destination: cdrRow[10],
-		SetupTime:   time.Date(2013, 2, 3, 19, 50, 0, 0, time.UTC),
-		AnswerTime:  time.Date(2013, 2, 3, 19, 54, 0, 0, time.UTC),
-		Usage:       time.Duration(62) * time.Second,
-		ExtraFields: map[string]string{"supplier": "supplier1"},
-		Cost:        -1,
-	}
-	if !reflect.DeepEqual(expectedCdr, rtCdr) {
-		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
+	if _, err := NewPartialFlatstoreRecord([]string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK"}); err == nil || err.Error() != "MISSING_IE" {
+		t.Error(err)
 	}
 }
+*/
 
 /*
-func TestDnTdmCdrs(t *testing.T) {
-	tdmCdrs := `
-49773280254,0049LN130676000285,N_IP_0676_00-Internet 0676 WRAP 13,02.07.2014 15:24:40,02.07.2014 15:24:40,1,25,Peak,0.000000,49DE13
-49893252121,0049651515477,N_MO_MRAP_00-WRAP Mobile,02.07.2014 15:24:41,02.07.2014 15:24:41,1,8,Peak,0.003920,49651
-49497361022,0049LM0409005226,N_MO_MTMB_00-RW-Mobile,02.07.2014 15:24:41,02.07.2014 15:24:41,1,43,Peak,0.021050,49MTMB
+func TestOsipsFlatstoreCdrs(t *testing.T) {
+	flatstoreCdrs := `
+INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475
+BYE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454410|||||3401:2069362475
+INVITE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454647|*postpaid|1002|1001||1877:893549741
+BYE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454651|||||1877:893549741
+INVITE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454657|*prepaid|1001|1002||2407:1884881533
+BYE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454661|||||2407:1884881533
+INVITE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454690|*prepaid|1001|1002||3099:1909036290
+BYE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454692|||||3099:1909036290
 `
-	cgrConfig, _ := config.NewDefaultCGRConfig()
-	eCdrs := []*utils.StoredCdr{
-		&utils.StoredCdr{
-			CgrId:       utils.Sha1("49773280254", time.Date(2014, 7, 2, 15, 24, 40, 0, time.UTC).String()),
-			TOR:         utils.VOICE,
-			AccId:       "49773280254",
-			CdrHost:     "0.0.0.0",
-			CdrSource:   cgrConfig.CdrcSourceId,
-			ReqType:     "rated",
-			Direction:   "*out",
-			Tenant:      "sip.test.deanconnect.nl",
-			Category:    "call",
-			Account:     "+49773280254",
-			Subject:     "+49773280254",
-			Destination: "+49676000285",
-			SetupTime:   time.Date(2014, 7, 2, 15, 24, 40, 0, time.UTC),
-			AnswerTime:  time.Date(2014, 7, 2, 15, 24, 40, 0, time.UTC),
-			Usage:       time.Duration(25) * time.Second,
-			Cost:        -1,
+
+	eCdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{
+			CgrId:           "e61034c34148a7c4f40623e00ca5e551d1408bf3",
+			TOR:             utils.VOICE,
+			AccId:           "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:02daec40c548625ac",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 06, 48, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 06, 48, 0, time.UTC),
+			Usage:           time.Duration(2) * time.Second,
+			DisconnectCause: "200 OK",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "3401:2069362475",
+			},
+			Cost: -1,
 		},
-		&utils.StoredCdr{
-			CgrId:       utils.Sha1("49893252121", time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC).String()),
-			TOR:         utils.VOICE,
-			AccId:       "49893252121",
-			CdrHost:     "0.0.0.0",
-			CdrSource:   cgrConfig.CdrcSourceId,
-			ReqType:     "rated",
-			Direction:   "*out",
-			Tenant:      "sip.test.deanconnect.nl",
-			Category:    "call",
-			Account:     "+49893252121",
-			Subject:     "+49893252121",
-			Destination: "+49651515477",
-			SetupTime:   time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC),
-			AnswerTime:  time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC),
-			Usage:       time.Duration(8) * time.Second,
-			Cost:        -1,
+		&engine.StoredCdr{
+			CgrId:           "3ed64a28190e20ac8a6fd8fd48cb23efbfeb7a17",
+			TOR:             utils.VOICE,
+			AccId:           "214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0f9d3d5c3c863a6e3",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_POSTPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1002",
+			Subject:         "1002",
+			Destination:     "1001",
+			SetupTime:       time.Date(2015, 7, 9, 15, 10, 47, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 10, 47, 0, time.UTC),
+			Usage:           time.Duration(4) * time.Second,
+			DisconnectCause: "200 OK",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "1877:893549741",
+			},
+			Cost: -1,
 		},
-		&utils.StoredCdr{
-			CgrId:       utils.Sha1("49497361022", time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC).String()),
-			TOR:         utils.VOICE,
-			AccId:       "49497361022",
-			CdrHost:     "0.0.0.0",
-			CdrSource:   cgrConfig.CdrcSourceId,
-			ReqType:     "rated",
-			Direction:   "*out",
-			Tenant:      "sip.test.deanconnect.nl",
-			Category:    "call",
-			Account:     "+49497361022",
-			Subject:     "+49497361022",
-			Destination: "+499005226",
-			SetupTime:   time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC),
-			AnswerTime:  time.Date(2014, 7, 2, 15, 24, 41, 0, time.UTC),
-			Usage:       time.Duration(43) * time.Second,
-			Cost:        -1,
+		&engine.StoredCdr{
+			CgrId:           "f2f8d9341adfbbe1836b22f75182142061ef3d20",
+			TOR:             utils.VOICE,
+			AccId:           "3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:036e39a542d996f9",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 10, 57, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 10, 57, 0, time.UTC),
+			Usage:           time.Duration(4) * time.Second,
+			DisconnectCause: "200 OK",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "2407:1884881533",
+			},
+			Cost: -1,
+		},
+		&engine.StoredCdr{
+			CgrId:           "ccf05e7e3b9db9d2370bcbe316817447dba7df54",
+			TOR:             utils.VOICE,
+			AccId:           "a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:03111f3c949ca4c42",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 11, 30, 0, time.UTC), //2015-07-09T17:11:30+02:00
+			AnswerTime:      time.Date(2015, 7, 9, 15, 11, 30, 0, time.UTC),
+			Usage:           time.Duration(2) * time.Second,
+			DisconnectCause: "200 OK",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "3099:1909036290",
+			},
+			Cost: -1,
 		},
 	}
-	torFld, _ := utils.NewRSRField("^*voice")
-	acntFld, _ := utils.NewRSRField(`~0:s/^([1-9]\d+)$/+$1/`)
-	reqTypeFld, _ := utils.NewRSRField("^rated")
-	dirFld, _ := utils.NewRSRField("^*out")
-	tenantFld, _ := utils.NewRSRField("^sip.test.deanconnect.nl")
-	categFld, _ := utils.NewRSRField("^call")
-	dstFld, _ := utils.NewRSRField(`~1:s/^00(\d+)(?:[a-zA-Z].{3})*0*([1-9]\d+)$/+$1$2/`)
-	usageFld, _ := utils.NewRSRField(`~6:s/^(\d+)$/${1}s/`)
-	cgrConfig.CdrcCdrFields = map[string]*utils.RSRField{
-		utils.TOR:         torFld,
-		utils.ACCID:       &utils.RSRField{Id: "0"},
-		utils.REQTYPE:     reqTypeFld,
-		utils.DIRECTION:   dirFld,
-		utils.TENANT:      tenantFld,
-		utils.CATEGORY:    categFld,
-		utils.ACCOUNT:     acntFld,
-		utils.SUBJECT:     acntFld,
-		utils.DESTINATION: dstFld,
-		utils.SETUP_TIME:  &utils.RSRField{Id: "4"},
-		utils.ANSWER_TIME: &utils.RSRField{Id: "4"},
-		utils.USAGE:       usageFld,
-	}
-	cdrc := &Cdrc{cgrConfig.CdrcCdrs, cgrConfig.CdrcCdrType, cgrConfig.CdrcCdrInDir, cgrConfig.CdrcCdrOutDir, cgrConfig.CdrcSourceId, cgrConfig.CdrcRunDelay, ',',
-		cgrConfig.CdrcCdrFields, new(cdrs.CDRS), nil}
-	cdrsContent := bytes.NewReader([]byte(tdmCdrs))
+
+	cdrFields := [][]*config.CfgCdrField{[]*config.CfgCdrField{
+		&config.CfgCdrField{Tag: "Tor", Type: utils.CDRFIELD, CdrFieldId: utils.TOR, Value: utils.ParseRSRFieldsMustCompile("^*voice", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "AccId", Type: utils.CDRFIELD, CdrFieldId: utils.ACCID, Mandatory: true},
+		&config.CfgCdrField{Tag: "ReqType", Type: utils.CDRFIELD, CdrFieldId: utils.REQTYPE, Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Tenant", Type: utils.CDRFIELD, CdrFieldId: utils.TENANT, Value: utils.ParseRSRFieldsMustCompile("^cgrates.org", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Category", Type: utils.CDRFIELD, CdrFieldId: utils.CATEGORY, Value: utils.ParseRSRFieldsMustCompile("^call", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Account", Type: utils.CDRFIELD, CdrFieldId: utils.ACCOUNT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Subject", Type: utils.CDRFIELD, CdrFieldId: utils.SUBJECT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Destination", Type: utils.CDRFIELD, CdrFieldId: utils.DESTINATION, Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "SetupTime", Type: utils.CDRFIELD, CdrFieldId: utils.SETUP_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "AnswerTime", Type: utils.CDRFIELD, CdrFieldId: utils.ANSWER_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Duration", Type: utils.CDRFIELD, CdrFieldId: utils.USAGE, Mandatory: true},
+		&config.CfgCdrField{Tag: "DisconnectCause", Type: utils.CDRFIELD, CdrFieldId: utils.DISCONNECT_CAUSE, Value: utils.ParseRSRFieldsMustCompile("4;^ ;5", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "DialogId", Type: utils.CDRFIELD, CdrFieldId: "DialogIdentifier", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP)},
+	}}
+	cdrc := &Cdrc{CdrFormat: utils.OSIPS_FLATSTORE, cdrSourceIds: []string{"TEST_CDRC"}, failedCallsPrefix: "missed_calls",
+		cdrFields: cdrFields, partialRecords: make(map[string]map[string]*PartialFlatstoreRecord),
+		guard: engine.NewGuardianLock()}
+	cdrsContent := bytes.NewReader([]byte(flatstoreCdrs))
 	csvReader := csv.NewReader(cdrsContent)
-	cdrs := make([]*utils.StoredCdr, 0)
+	csvReader.Comma = '|'
+	cdrs := make([]*engine.StoredCdr, 0)
+	recNrs := 0
 	for {
+		recNrs++
 		cdrCsv, err := csvReader.Read()
 		if err != nil && err == io.EOF {
 			break // End of file
 		} else if err != nil {
 			t.Error("Unexpected error:", err)
 		}
-		if cdr, err := cdrc.recordToStoredCdr(cdrCsv); err != nil {
-			t.Error("Unexpected error: ", err)
-		} else {
-			cdrs = append(cdrs, cdr)
+		record, err := cdrc.processPartialRecord(cdrCsv, "dummyfilename")
+		if err != nil {
+			t.Error(err)
+		}
+		if record == nil {
+			continue // Partial record
+		}
+		if storedCdr, err := cdrc.recordToStoredCdr(record, 0); err != nil {
+			t.Error(err)
+		} else if storedCdr != nil {
+			cdrs = append(cdrs, storedCdr)
 		}
 	}
 	if !reflect.DeepEqual(eCdrs, cdrs) {
-		for _, ecdr := range eCdrs {
-			fmt.Printf("Cdr expected: %+v\n", ecdr)
+		t.Errorf("Expecting: %+v, received: %+v", eCdrs, cdrs)
+	}
+
+}
+
+func TestOsipsFlatstoreMissedCdrs(t *testing.T) {
+	flatstoreCdrs := `
+INVITE|ef6c6256|da501581|0bfdd176d1b93e7df3de5c6f4873ee04@0:0:0:0:0:0:0:0|487|Request Terminated|1436454643|*prepaid|1001|1002||1224:339382783
+INVITE|7905e511||81880da80a94bda81b425b09009e055c@0:0:0:0:0:0:0:0|404|Not Found|1436454668|*prepaid|1001|1002||1980:1216490844
+INVITE|324cb497|d4af7023|8deaadf2ae9a17809a391f05af31afb0@0:0:0:0:0:0:0:0|486|Busy here|1436454687|*postpaid|1002|1001||474:130115066
+`
+	eCdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{
+			CgrId:           "1c20aa6543a5a30d26b2354ae79e1f5fb720e8e5",
+			TOR:             utils.VOICE,
+			AccId:           "0bfdd176d1b93e7df3de5c6f4873ee04@0:0:0:0:0:0:0:0ef6c6256da501581",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 10, 43, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 10, 43, 0, time.UTC),
+			Usage:           0,
+			DisconnectCause: "487 Request Terminated",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "1224:339382783",
+			},
+			Cost: -1,
+		},
+		&engine.StoredCdr{
+			CgrId:           "054ab7c6c7fe6dc4a72f34e270027fa2aa930a58",
+			TOR:             utils.VOICE,
+			AccId:           "81880da80a94bda81b425b09009e055c@0:0:0:0:0:0:0:07905e511",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 11, 8, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 11, 8, 0, time.UTC),
+			Usage:           0,
+			DisconnectCause: "404 Not Found",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "1980:1216490844",
+			},
+			Cost: -1,
+		},
+		&engine.StoredCdr{
+			CgrId:           "d49ea63d1655b15149336004629f1cadd1434b89",
+			TOR:             utils.VOICE,
+			AccId:           "8deaadf2ae9a17809a391f05af31afb0@0:0:0:0:0:0:0:0324cb497d4af7023",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_POSTPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1002",
+			Subject:         "1002",
+			Destination:     "1001",
+			SetupTime:       time.Date(2015, 7, 9, 15, 11, 27, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 11, 27, 0, time.UTC),
+			Usage:           0,
+			DisconnectCause: "486 Busy here",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "474:130115066",
+			},
+			Cost: -1,
+		},
+	}
+
+	cdrFields := [][]*config.CfgCdrField{[]*config.CfgCdrField{
+		&config.CfgCdrField{Tag: "Tor", Type: utils.CDRFIELD, CdrFieldId: utils.TOR, Value: utils.ParseRSRFieldsMustCompile("^*voice", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "AccId", Type: utils.CDRFIELD, CdrFieldId: utils.ACCID, Mandatory: true},
+		&config.CfgCdrField{Tag: "ReqType", Type: utils.CDRFIELD, CdrFieldId: utils.REQTYPE, Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Tenant", Type: utils.CDRFIELD, CdrFieldId: utils.TENANT, Value: utils.ParseRSRFieldsMustCompile("^cgrates.org", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Category", Type: utils.CDRFIELD, CdrFieldId: utils.CATEGORY, Value: utils.ParseRSRFieldsMustCompile("^call", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Account", Type: utils.CDRFIELD, CdrFieldId: utils.ACCOUNT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Subject", Type: utils.CDRFIELD, CdrFieldId: utils.SUBJECT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Destination", Type: utils.CDRFIELD, CdrFieldId: utils.DESTINATION, Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "SetupTime", Type: utils.CDRFIELD, CdrFieldId: utils.SETUP_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "AnswerTime", Type: utils.CDRFIELD, CdrFieldId: utils.ANSWER_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Usage", Type: utils.CDRFIELD, CdrFieldId: utils.USAGE, Mandatory: true},
+		&config.CfgCdrField{Tag: "DisconnectCause", Type: utils.CDRFIELD, CdrFieldId: utils.DISCONNECT_CAUSE, Value: utils.ParseRSRFieldsMustCompile("4;^ ;5", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "DialogId", Type: utils.CDRFIELD, CdrFieldId: "DialogIdentifier", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP)},
+	}}
+	cdrc := &Cdrc{CdrFormat: utils.OSIPS_FLATSTORE, cdrSourceIds: []string{"TEST_CDRC"}, failedCallsPrefix: "missed_calls",
+		cdrFields: cdrFields, partialRecords: make(map[string]map[string]*PartialFlatstoreRecord),
+		guard: engine.NewGuardianLock()}
+	cdrsContent := bytes.NewReader([]byte(flatstoreCdrs))
+	csvReader := csv.NewReader(cdrsContent)
+	csvReader.Comma = '|'
+	cdrs := make([]*engine.StoredCdr, 0)
+	recNrs := 0
+	for {
+		recNrs++
+		cdrCsv, err := csvReader.Read()
+		if err != nil && err == io.EOF {
+			break // End of file
+		} else if err != nil {
+			t.Error("Unexpected error:", err)
 		}
-		for _, cdr := range cdrs {
-			fmt.Printf("Cdr processed: %+v\n", cdr)
+		record, err := cdrc.processPartialRecord(cdrCsv, "missed_calls_1.log")
+		if err != nil {
+			t.Error(err)
 		}
+		if record == nil {
+			continue // Partial record
+		}
+		if storedCdr, err := cdrc.recordToStoredCdr(record, 0); err != nil {
+			t.Error(err)
+		} else if storedCdr != nil {
+			cdrs = append(cdrs, storedCdr)
+		}
+	}
+	if !reflect.DeepEqual(eCdrs, cdrs) {
 		t.Errorf("Expecting: %+v, received: %+v", eCdrs, cdrs)
 	}
 

cdrc/csv.go

@@ -0,0 +1,334 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func NewPartialFlatstoreRecord(record []string) (*PartialFlatstoreRecord, error) {
+	if len(record) < 7 {
+		return nil, errors.New("MISSING_IE")
+	}
+	pr := &PartialFlatstoreRecord{Method: record[0], AccId: record[3] + record[1] + record[2], Values: record}
+	var err error
+	if pr.Timestamp, err = utils.ParseTimeDetectLayout(record[6]); err != nil {
+		return nil, err
+	}
+	return pr, nil
+}
+
+// This is a partial record received from Flatstore, can be INVITE or BYE and it needs to be paired in order to produce duration
+type PartialFlatstoreRecord struct {
+	Method    string    // INVITE or BYE
+	AccId     string    // Copute here the AccId
+	Timestamp time.Time // Timestamp of the event, as written by db_flastore module
+	Values    []string  // Can contain original values or updated via UpdateValues
+}
+
+// Pairs INVITE and BYE into final record containing as last element the duration
+func pairToRecord(part1, part2 *PartialFlatstoreRecord) ([]string, error) {
+	var invite, bye *PartialFlatstoreRecord
+	if part1.Method == "INVITE" {
+		invite = part1
+	} else if part2.Method == "INVITE" {
+		invite = part2
+	} else {
+		return nil, errors.New("MISSING_INVITE")
+	}
+	if part1.Method == "BYE" {
+		bye = part1
+	} else if part2.Method == "BYE" {
+		bye = part2
+	} else {
+		return nil, errors.New("MISSING_BYE")
+	}
+	if len(invite.Values) != len(bye.Values) {
+		return nil, errors.New("INCONSISTENT_VALUES_LENGTH")
+	}
+	record := invite.Values
+	for idx := range record {
+		switch idx {
+		case 0, 1, 2, 3, 6: // Leave these values as they are
+		case 4, 5:
+			record[idx] = bye.Values[idx] // Update record with status from bye
+		default:
+			if bye.Values[idx] != "" { // Any value higher than 6 is dynamically inserted, overwrite if non empty
+				record[idx] = bye.Values[idx]
+			}
+
+		}
+	}
+	callDur := bye.Timestamp.Sub(invite.Timestamp)
+	record = append(record, strconv.FormatFloat(callDur.Seconds(), 'f', -1, 64))
+	return record, nil
+}
+
+func NewPartialRecordsCache(ttl time.Duration, cdrOutDir string, csvSep rune) (*PartialRecordsCache, error) {
+	return &PartialRecordsCache{ttl: ttl, cdrOutDir: cdrOutDir, csvSep: csvSep,
+		partialRecords: make(map[string]map[string]*PartialFlatstoreRecord), guard: engine.NewGuardianLock()}, nil
+}
+
+type PartialRecordsCache struct {
+	ttl            time.Duration
+	cdrOutDir      string
+	csvSep         rune
+	partialRecords map[string]map[string]*PartialFlatstoreRecord // [FileName"][AccId]*PartialRecord
+	guard          *engine.GuardianLock
+}
+
+// Dumps the cache into a .unpaired file in the outdir and cleans cache after
+func (self *PartialRecordsCache) dumpUnpairedRecords(fileName string) error {
+	_, err := self.guard.Guard(func() (interface{}, error) {
+		if len(self.partialRecords[fileName]) != 0 { // Only write the file if there are records in the cache
+			unpairedFilePath := path.Join(self.cdrOutDir, fileName+UNPAIRED_SUFFIX)
+			fileOut, err := os.Create(unpairedFilePath)
+			if err != nil {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed creating %s, error: %s", unpairedFilePath, err.Error()))
+				return nil, err
+			}
+			csvWriter := csv.NewWriter(fileOut)
+			csvWriter.Comma = self.csvSep
+			for _, pr := range self.partialRecords[fileName] {
+				if err := csvWriter.Write(pr.Values); err != nil {
+					engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed writing unpaired record %v to file: %s, error: %s", pr, unpairedFilePath, err.Error()))
+					return nil, err
+				}
+			}
+			csvWriter.Flush()
+		}
+		delete(self.partialRecords, fileName)
+		return nil, nil
+	}, fileName)
+	return err
+}
+
+// Search in cache and return the partial record with accountind id defined, prefFilename is searched at beginning because of better match probability
+func (self *PartialRecordsCache) GetPartialRecord(accId, prefFileName string) (string, *PartialFlatstoreRecord) {
+	var cachedFilename string
+	var cachedPartial *PartialFlatstoreRecord
+	checkCachedFNames := []string{prefFileName} // Higher probability to match as firstFileName
+	for fName := range self.partialRecords {
+		if fName != prefFileName {
+			checkCachedFNames = append(checkCachedFNames, fName)
+		}
+	}
+	for _, fName := range checkCachedFNames { // Need to lock them individually
+		self.guard.Guard(func() (interface{}, error) {
+			var hasPartial bool
+			if cachedPartial, hasPartial = self.partialRecords[fName][accId]; hasPartial {
+				cachedFilename = fName
+			}
+			return nil, nil
+		}, fName)
+		if cachedPartial != nil {
+			break
+		}
+	}
+	return cachedFilename, cachedPartial
+}
+
+func (self *PartialRecordsCache) CachePartial(fileName string, pr *PartialFlatstoreRecord) {
+	self.guard.Guard(func() (interface{}, error) {
+		if fileMp, hasFile := self.partialRecords[fileName]; !hasFile {
+			self.partialRecords[fileName] = map[string]*PartialFlatstoreRecord{pr.AccId: pr}
+			if self.ttl != 0 { // Schedule expiry/dump of the just created entry in cache
+				go func() {
+					time.Sleep(self.ttl)
+					self.dumpUnpairedRecords(fileName)
+				}()
+			}
+		} else if _, hasAccId := fileMp[pr.AccId]; !hasAccId {
+			self.partialRecords[fileName][pr.AccId] = pr
+		}
+		return nil, nil
+	}, fileName)
+}
+
+func (self *PartialRecordsCache) UncachePartial(fileName string, pr *PartialFlatstoreRecord) {
+	self.guard.Guard(func() (interface{}, error) {
+		delete(self.partialRecords[fileName], pr.AccId) // Remove the record out of cache
+		return nil, nil
+	}, fileName)
+}
+
+func NewCsvRecordsProcessor(csvReader *csv.Reader, cdrFormat, fileName, failedCallsPrefix string,
+	cdrSourceIds []string, duMultiplyFactors []float64, cdrFilters []utils.RSRFields, cdrFields [][]*config.CfgCdrField,
+	httpSkipTlsCheck bool, partialRecordsCache *PartialRecordsCache) *CsvRecordsProcessor {
+	return &CsvRecordsProcessor{csvReader: csvReader, cdrFormat: cdrFormat, fileName: fileName,
+		failedCallsPrefix: failedCallsPrefix, cdrSourceIds: cdrSourceIds,
+		duMultiplyFactors: duMultiplyFactors, cdrFilters: cdrFilters, cdrFields: cdrFields,
+		httpSkipTlsCheck: httpSkipTlsCheck, partialRecordsCache: partialRecordsCache}
+
+}
+
+type CsvRecordsProcessor struct {
+	csvReader           *csv.Reader
+	cdrFormat           string
+	fileName            string
+	failedCallsPrefix   string
+	cdrSourceIds        []string // Should be in sync with cdrFields on indexes
+	duMultiplyFactors   []float64
+	cdrFilters          []utils.RSRFields       // Should be in sync with cdrFields on indexes
+	cdrFields           [][]*config.CfgCdrField // Profiles directly connected with cdrFilters
+	httpSkipTlsCheck    bool
+	partialRecordsCache *PartialRecordsCache // Shared by cdrc so we can cache for all files in a folder
+}
+
+func (self *CsvRecordsProcessor) ProcessNextRecord() ([]*engine.StoredCdr, error) {
+	record, err := self.csvReader.Read()
+	if err != nil {
+		return nil, err
+	}
+	if utils.IsSliceMember([]string{utils.KAM_FLATSTORE, utils.OSIPS_FLATSTORE}, self.cdrFormat) {
+		if record, err = self.processPartialRecord(record); err != nil {
+			return nil, err
+		} else if record == nil {
+			return nil, nil // Due to partial, none returned
+		}
+	}
+	// Record was overwriten with complete information out of cache
+	return self.processRecord(record)
+}
+
+// Processes a single partial record for flatstore CDRs
+func (self *CsvRecordsProcessor) processPartialRecord(record []string) ([]string, error) {
+	if strings.HasPrefix(self.fileName, self.failedCallsPrefix) { // Use the first index since they should be the same in all configs
+		record = append(record, "0") // Append duration 0 for failed calls flatstore CDR and do not process it further
+		return record, nil
+	}
+	pr, err := NewPartialFlatstoreRecord(record)
+	if err != nil {
+		return nil, err
+	}
+	// Retrieve and complete the record from cache
+	cachedFilename, cachedPartial := self.partialRecordsCache.GetPartialRecord(pr.AccId, self.fileName)
+	if cachedPartial == nil { // Not cached, do it here and stop processing
+		self.partialRecordsCache.CachePartial(self.fileName, pr)
+		return nil, nil
+	}
+	pairedRecord, err := pairToRecord(cachedPartial, pr)
+	if err != nil {
+		return nil, err
+	}
+	self.partialRecordsCache.UncachePartial(cachedFilename, pr)
+	return pairedRecord, nil
+}
+
+// Takes the record from a slice and turns it into StoredCdrs, posting them to the cdrServer
+func (self *CsvRecordsProcessor) processRecord(record []string) ([]*engine.StoredCdr, error) {
+	recordCdrs := make([]*engine.StoredCdr, 0) // More CDRs based on the number of filters and field templates
+	for idx := range self.cdrFields {          // cdrFields coming from more templates will produce individual storCdr records
+		// Make sure filters are matching
+		filterBreak := false
+		for _, rsrFilter := range self.cdrFilters[idx] {
+			if rsrFilter == nil { // Nil filter does not need to match anything
+				continue
+			}
+			if cfgFieldIdx, _ := strconv.Atoi(rsrFilter.Id); len(record) <= cfgFieldIdx {
+				return nil, fmt.Errorf("Ignoring record: %v - cannot compile filter %+v", record, rsrFilter)
+			} else if !rsrFilter.FilterPasses(record[cfgFieldIdx]) {
+				filterBreak = true
+				break
+			}
+		}
+		if filterBreak { // Stop importing cdrc fields profile due to non matching filter
+			continue
+		}
+		if storedCdr, err := self.recordToStoredCdr(record, idx); err != nil {
+			return nil, fmt.Errorf("Failed converting to StoredCdr, error: %s", err.Error())
+		} else {
+			recordCdrs = append(recordCdrs, storedCdr)
+		}
+	}
+	return recordCdrs, nil
+}
+
+// Takes the record out of csv and turns it into storedCdr which can be processed by CDRS
+func (self *CsvRecordsProcessor) recordToStoredCdr(record []string, cfgIdx int) (*engine.StoredCdr, error) {
+	storedCdr := &engine.StoredCdr{CdrHost: "0.0.0.0", CdrSource: self.cdrSourceIds[cfgIdx], ExtraFields: make(map[string]string), Cost: -1}
+	var err error
+	var lazyHttpFields []*config.CfgCdrField
+	for _, cdrFldCfg := range self.cdrFields[cfgIdx] {
+		if utils.IsSliceMember([]string{utils.KAM_FLATSTORE, utils.OSIPS_FLATSTORE}, self.cdrFormat) { // Hardcode some values in case of flatstore
+			switch cdrFldCfg.CdrFieldId {
+			case utils.ACCID:
+				cdrFldCfg.Value = utils.ParseRSRFieldsMustCompile("3;1;2", utils.INFIELD_SEP) // in case of flatstore, accounting id is made up out of callid, from_tag and to_tag
+			case utils.USAGE:
+				cdrFldCfg.Value = utils.ParseRSRFieldsMustCompile(strconv.Itoa(len(record)-1), utils.INFIELD_SEP) // in case of flatstore, last element will be the duration computed by us
+			}
+
+		}
+		var fieldVal string
+		if cdrFldCfg.Type == utils.CDRFIELD {
+			for _, cfgFieldRSR := range cdrFldCfg.Value {
+				if cfgFieldRSR.IsStatic() {
+					fieldVal += cfgFieldRSR.ParseValue("")
+				} else { // Dynamic value extracted using index
+					if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx {
+						return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cdrFldCfg.Tag)
+					} else {
+						fieldVal += cfgFieldRSR.ParseValue(record[cfgFieldIdx])
+					}
+				}
+			}
+		} else if cdrFldCfg.Type == utils.HTTP_POST {
+			lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of storedCdr to http server
+		} else {
+			return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type)
+		}
+		if err := populateStoredCdrField(storedCdr, cdrFldCfg.CdrFieldId, fieldVal); err != nil {
+			return nil, err
+		}
+	}
+	storedCdr.CgrId = utils.Sha1(storedCdr.AccId, storedCdr.SetupTime.String())
+	if storedCdr.TOR == utils.DATA && self.duMultiplyFactors[cfgIdx] != 0 {
+		storedCdr.Usage = time.Duration(float64(storedCdr.Usage.Nanoseconds()) * self.duMultiplyFactors[cfgIdx])
+	}
+	for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields
+		var outValByte []byte
+		var fieldVal, httpAddr string
+		for _, rsrFld := range httpFieldCfg.Value {
+			httpAddr += rsrFld.ParseValue("")
+		}
+		if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, storedCdr); err != nil && httpFieldCfg.Mandatory {
+			return nil, err
+		} else {
+			fieldVal = string(outValByte)
+			if len(fieldVal) == 0 && httpFieldCfg.Mandatory {
+				return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag)
+			}
+			if err := populateStoredCdrField(storedCdr, httpFieldCfg.CdrFieldId, fieldVal); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return storedCdr, nil
+}

cdrc/csv_test.go

@@ -0,0 +1,151 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func TestCsvRecordForkCdr(t *testing.T) {
+	cgrConfig, _ := config.NewDefaultCGRConfig()
+	cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][utils.META_DEFAULT]
+	cdrcConfig.ContentFields = append(cdrcConfig.ContentFields, &config.CfgCdrField{Tag: "SupplierTest", Type: utils.CDRFIELD, CdrFieldId: "supplier", Value: []*utils.RSRField{&utils.RSRField{Id: "14"}}})
+	cdrcConfig.ContentFields = append(cdrcConfig.ContentFields, &config.CfgCdrField{Tag: "DisconnectCauseTest", Type: utils.CDRFIELD, CdrFieldId: utils.DISCONNECT_CAUSE,
+		Value: []*utils.RSRField{&utils.RSRField{Id: "16"}}})
+	csvProcessor := &CsvRecordsProcessor{cdrFormat: CSV, cdrSourceIds: []string{"TEST_CDRC"}, cdrFields: [][]*config.CfgCdrField{cdrcConfig.ContentFields}}
+	cdrRow := []string{"firstField", "secondField"}
+	_, err := csvProcessor.recordToStoredCdr(cdrRow, 0)
+	if err == nil {
+		t.Error("Failed to corectly detect missing fields from record")
+	}
+	cdrRow = []string{"ignored", "ignored", utils.VOICE, "acc1", utils.META_PREPAID, "*out", "cgrates.org", "call", "1001", "1001", "+4986517174963",
+		"2013-02-03 19:50:00", "2013-02-03 19:54:00", "62", "supplier1", "172.16.1.1", "NORMAL_DISCONNECT"}
+	rtCdr, err := csvProcessor.recordToStoredCdr(cdrRow, 0)
+	if err != nil {
+		t.Error("Failed to parse CDR in rated cdr", err)
+	}
+	expectedCdr := &engine.StoredCdr{
+		CgrId:           utils.Sha1(cdrRow[3], time.Date(2013, 2, 3, 19, 50, 0, 0, time.UTC).String()),
+		TOR:             cdrRow[2],
+		AccId:           cdrRow[3],
+		CdrHost:         "0.0.0.0", // Got it over internal interface
+		CdrSource:       "TEST_CDRC",
+		ReqType:         cdrRow[4],
+		Direction:       cdrRow[5],
+		Tenant:          cdrRow[6],
+		Category:        cdrRow[7],
+		Account:         cdrRow[8],
+		Subject:         cdrRow[9],
+		Destination:     cdrRow[10],
+		SetupTime:       time.Date(2013, 2, 3, 19, 50, 0, 0, time.UTC),
+		AnswerTime:      time.Date(2013, 2, 3, 19, 54, 0, 0, time.UTC),
+		Usage:           time.Duration(62) * time.Second,
+		Supplier:        "supplier1",
+		DisconnectCause: "NORMAL_DISCONNECT",
+		ExtraFields:     map[string]string{},
+		Cost:            -1,
+	}
+	if !reflect.DeepEqual(expectedCdr, rtCdr) {
+		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
+	}
+}
+
+func TestCsvDataMultiplyFactor(t *testing.T) {
+	cdrFields := []*config.CfgCdrField{&config.CfgCdrField{Tag: "TORField", Type: utils.CDRFIELD, CdrFieldId: "tor", Value: []*utils.RSRField{&utils.RSRField{Id: "0"}}},
+		&config.CfgCdrField{Tag: "UsageField", Type: utils.CDRFIELD, CdrFieldId: "usage", Value: []*utils.RSRField{&utils.RSRField{Id: "1"}}}}
+	csvProcessor := &CsvRecordsProcessor{cdrFormat: CSV, cdrSourceIds: []string{"TEST_CDRC"}, duMultiplyFactors: []float64{0}, cdrFields: [][]*config.CfgCdrField{cdrFields}}
+	cdrRow := []string{"*data", "1"}
+	rtCdr, err := csvProcessor.recordToStoredCdr(cdrRow, 0)
+	if err != nil {
+		t.Error("Failed to parse CDR in rated cdr", err)
+	}
+	var sTime time.Time
+	expectedCdr := &engine.StoredCdr{
+		CgrId:       utils.Sha1("", sTime.String()),
+		TOR:         cdrRow[0],
+		CdrHost:     "0.0.0.0",
+		CdrSource:   "TEST_CDRC",
+		Usage:       time.Duration(1) * time.Second,
+		ExtraFields: map[string]string{},
+		Cost:        -1,
+	}
+	if !reflect.DeepEqual(expectedCdr, rtCdr) {
+		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
+	}
+	csvProcessor.duMultiplyFactors = []float64{1024}
+	expectedCdr = &engine.StoredCdr{
+		CgrId:       utils.Sha1("", sTime.String()),
+		TOR:         cdrRow[0],
+		CdrHost:     "0.0.0.0",
+		CdrSource:   "TEST_CDRC",
+		Usage:       time.Duration(1024) * time.Second,
+		ExtraFields: map[string]string{},
+		Cost:        -1,
+	}
+	if rtCdr, _ := csvProcessor.recordToStoredCdr(cdrRow, 0); !reflect.DeepEqual(expectedCdr, rtCdr) {
+		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
+	}
+	cdrRow = []string{"*voice", "1"}
+	expectedCdr = &engine.StoredCdr{
+		CgrId:       utils.Sha1("", sTime.String()),
+		TOR:         cdrRow[0],
+		CdrHost:     "0.0.0.0",
+		CdrSource:   "TEST_CDRC",
+		Usage:       time.Duration(1) * time.Second,
+		ExtraFields: map[string]string{},
+		Cost:        -1,
+	}
+	if rtCdr, _ := csvProcessor.recordToStoredCdr(cdrRow, 0); !reflect.DeepEqual(expectedCdr, rtCdr) {
+		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
+	}
+}
+
+func TestCsvPairToRecord(t *testing.T) {
+	eRecord := []string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454408", "*prepaid", "1001", "1002", "", "3401:2069362475", "2"}
+	invPr := &PartialFlatstoreRecord{Method: "INVITE", Timestamp: time.Date(2015, 7, 9, 15, 6, 48, 0, time.UTC),
+		Values: []string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454408", "*prepaid", "1001", "1002", "", "3401:2069362475"}}
+	byePr := &PartialFlatstoreRecord{Method: "BYE", Timestamp: time.Date(2015, 7, 9, 15, 6, 50, 0, time.UTC),
+		Values: []string{"BYE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454410", "", "", "", "", "3401:2069362475"}}
+	if rec, err := pairToRecord(invPr, byePr); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eRecord, rec) {
+		t.Errorf("Expected: %+v, received: %+v", eRecord, rec)
+	}
+	if rec, err := pairToRecord(byePr, invPr); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eRecord, rec) {
+		t.Errorf("Expected: %+v, received: %+v", eRecord, rec)
+	}
+	if _, err := pairToRecord(byePr, byePr); err == nil || err.Error() != "MISSING_INVITE" {
+		t.Error(err)
+	}
+	if _, err := pairToRecord(invPr, invPr); err == nil || err.Error() != "MISSING_BYE" {
+		t.Error(err)
+	}
+	byePr.Values = []string{"BYE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454410", "", "", "", "3401:2069362475"} // Took one value out
+	if _, err := pairToRecord(invPr, byePr); err == nil || err.Error() != "INCONSISTENT_VALUES_LENGTH" {
+		t.Error(err)
+	}
+}

cdrc/flatstore_local_test.go

@@ -0,0 +1,161 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"io/ioutil"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+)
+
+var flatstoreCfgPath string
+var flatstoreCfg *config.CGRConfig
+var flatstoreRpc *rpc.Client
+var flatstoreCdrcCfg *config.CdrcConfig
+
+var fullSuccessfull = `INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475
+BYE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454410|||||3401:2069362475
+INVITE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454647|*postpaid|1002|1001||1877:893549741
+BYE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454651|||||1877:893549741
+INVITE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454657|*prepaid|1001|1002||2407:1884881533
+BYE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454661|||||2407:1884881533
+INVITE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454690|*prepaid|1001|1002||3099:1909036290
+BYE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454692|||||3099:1909036290
+`
+
+var fullMissed = `INVITE|ef6c6256|da501581|0bfdd176d1b93e7df3de5c6f4873ee04@0:0:0:0:0:0:0:0|487|Request Terminated|1436454643|*prepaid|1001|1002||1224:339382783
+INVITE|7905e511||81880da80a94bda81b425b09009e055c@0:0:0:0:0:0:0:0|404|Not Found|1436454668|*prepaid|1001|1002||1980:1216490844
+INVITE|324cb497|d4af7023|8deaadf2ae9a17809a391f05af31afb0@0:0:0:0:0:0:0:0|486|Busy here|1436454687|*postpaid|1002|1001||474:130115066`
+
+var part1 = `BYE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4ccb@0:0:0:0:0:0:0:0|200|OK|1436454651|||||1877:893549742
+`
+
+var part2 = `INVITE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4ccb@0:0:0:0:0:0:0:0|200|OK|1436454647|*postpaid|1002|1003||1877:893549742
+INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475`
+
+func TestFlatstoreLclInitCfg(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	flatstoreCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcflatstore")
+	if flatstoreCfg, err = config.NewCGRConfigFromFolder(flatstoreCfgPath); err != nil {
+		t.Fatal("Got config error: ", err.Error())
+	}
+}
+
+// InitDb so we can rely on count
+func TestFlatstoreLclInitCdrDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitStorDb(flatstoreCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Creates cdr files and moves them into processing folder
+func TestFlatstoreLclCreateCdrFiles(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if flatstoreCfg == nil {
+		t.Fatal("Empty default cdrc configuration")
+	}
+	flatstoreCdrcCfg = flatstoreCfg.CdrcProfiles["/tmp/cgr_flatstore/cdrc/in"]["FLATSTORE"]
+	if err := os.RemoveAll(flatstoreCdrcCfg.CdrInDir); err != nil {
+		t.Fatal("Error removing folder: ", flatstoreCdrcCfg.CdrInDir, err)
+	}
+	if err := os.MkdirAll(flatstoreCdrcCfg.CdrInDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", flatstoreCdrcCfg.CdrInDir, err)
+	}
+	if err := os.RemoveAll(flatstoreCdrcCfg.CdrOutDir); err != nil {
+		t.Fatal("Error removing folder: ", flatstoreCdrcCfg.CdrOutDir, err)
+	}
+	if err := os.MkdirAll(flatstoreCdrcCfg.CdrOutDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", flatstoreCdrcCfg.CdrOutDir, err)
+	}
+}
+
+func TestFlatstoreLclStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if _, err := engine.StopStartEngine(flatstoreCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestFlatstoreLclRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	flatstoreRpc, err = jsonrpc.Dial("tcp", flatstoreCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+func TestFlatstoreLclProcessFiles(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := ioutil.WriteFile(path.Join("/tmp", "acc_1.log"), []byte(fullSuccessfull), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	if err := ioutil.WriteFile(path.Join("/tmp", "missed_calls_1.log"), []byte(fullMissed), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	if err := ioutil.WriteFile(path.Join("/tmp", "acc_2.log"), []byte(part1), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	if err := ioutil.WriteFile(path.Join("/tmp", "acc_3.log"), []byte(part2), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	//Rename(oldpath, newpath string)
+	for _, fileName := range []string{"acc_1.log", "missed_calls_1.log", "acc_2.log", "acc_3.log"} {
+		if err := os.Rename(path.Join("/tmp", fileName), path.Join(flatstoreCdrcCfg.CdrInDir, fileName)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	time.Sleep(time.Duration(2) * time.Second) // Give time for processing to happen and the .unparired file to be written
+	filesInDir, _ := ioutil.ReadDir(flatstoreCdrcCfg.CdrInDir)
+	if len(filesInDir) != 0 {
+		t.Errorf("Files in cdrcInDir: %+v", filesInDir)
+	}
+	filesOutDir, _ := ioutil.ReadDir(flatstoreCdrcCfg.CdrOutDir)
+	if len(filesOutDir) != 5 {
+		t.Errorf("In CdrcOutDir, expecting 5 files, got: %d", len(filesOutDir))
+	}
+	ePartContent := "INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475\n"
+	if partContent, err := ioutil.ReadFile(path.Join(flatstoreCdrcCfg.CdrOutDir, "acc_3.log.unpaired")); err != nil {
+		t.Error(err)
+	} else if ePartContent != string(partContent) {
+		t.Errorf("Expecting: %s, received: %s", ePartContent, string(partContent))
+	}
+}

cdrc/fwv.go

@@ -0,0 +1,246 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func fwvValue(cdrLine string, indexStart, width int, padding string) string {
+	rawVal := cdrLine[indexStart : indexStart+width]
+	switch padding {
+	case "left":
+		rawVal = strings.TrimLeft(rawVal, " ")
+	case "right":
+		rawVal = strings.TrimRight(rawVal, " ")
+	case "zeroleft":
+		rawVal = strings.TrimLeft(rawVal, "0 ")
+	case "zeroright":
+		rawVal = strings.TrimRight(rawVal, "0 ")
+	}
+	return rawVal
+}
+
+func NewFwvRecordsProcessor(file *os.File, cdrcCfgs map[string]*config.CdrcConfig, dfltCfg *config.CdrcConfig, httpClient *http.Client, httpSkipTlsCheck bool) *FwvRecordsProcessor {
+	return &FwvRecordsProcessor{file: file, cdrcCfgs: cdrcCfgs, dfltCfg: dfltCfg, httpSkipTlsCheck: httpSkipTlsCheck}
+}
+
+type FwvRecordsProcessor struct {
+	file             *os.File
+	cdrcCfgs         map[string]*config.CdrcConfig
+	dfltCfg          *config.CdrcConfig // General parameters
+	httpClient       *http.Client
+	httpSkipTlsCheck bool
+	lineLen          int64             // Length of the line in the file
+	offset           int64             // Index of the next byte to process
+	trailerOffset    int64             // Index where trailer starts, to be used as boundary when reading cdrs
+	headerCdr        *engine.StoredCdr // Cache here the general purpose stored CDR
+}
+
+// Sets the line length based on first line, sets offset back to initial after reading
+func (self *FwvRecordsProcessor) setLineLen() error {
+	rdr := bufio.NewReader(self.file)
+	readBytes, err := rdr.ReadBytes('\n')
+	if err != nil {
+		return err
+	}
+	self.lineLen = int64(len(readBytes))
+	if _, err := self.file.Seek(0, 0); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (self *FwvRecordsProcessor) ProcessNextRecord() ([]*engine.StoredCdr, error) {
+	defer func() { self.offset += self.lineLen }() // Schedule increasing the offset once we are out from processing the record
+	if self.offset == 0 {                          // First time, set the necessary offsets
+		if err := self.setLineLen(); err != nil {
+			engine.Logger.Err(fmt.Sprintf("<Cdrc> Row 0, error: cannot set lineLen: %s", err.Error()))
+			return nil, io.EOF
+		}
+		if len(self.dfltCfg.TrailerFields) != 0 {
+			if fi, err := self.file.Stat(); err != nil {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Row 0, error: cannot get file stats: %s", err.Error()))
+				return nil, err
+			} else {
+				self.trailerOffset = fi.Size() - self.lineLen
+			}
+		}
+		if len(self.dfltCfg.HeaderFields) != 0 { // ToDo: Process here the header fields
+			if err := self.processHeader(); err != nil {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Row 0, error reading header: %s", err.Error()))
+				return nil, io.EOF
+			}
+			return nil, nil
+		}
+	}
+	recordCdrs := make([]*engine.StoredCdr, 0) // More CDRs based on the number of filters and field templates
+	if self.trailerOffset != 0 && self.offset >= self.trailerOffset {
+		if err := self.processTrailer(); err != nil && err != io.EOF {
+			engine.Logger.Err(fmt.Sprintf("<Cdrc> Read trailer error: %s ", err.Error()))
+		}
+		return nil, io.EOF
+	}
+	buf := make([]byte, self.lineLen)
+	nRead, err := self.file.Read(buf)
+	if err != nil {
+		return nil, err
+	} else if nRead != len(buf) {
+		engine.Logger.Err(fmt.Sprintf("<Cdrc> Could not read complete line, have instead: %s", string(buf)))
+		return nil, io.EOF
+	}
+	record := string(buf)
+	for cfgKey := range self.cdrcCfgs {
+		if passes := self.recordPassesCfgFilter(record, cfgKey); !passes {
+			continue
+		}
+		if storedCdr, err := self.recordToStoredCdr(record, cfgKey); err != nil {
+			return nil, fmt.Errorf("Failed converting to StoredCdr, error: %s", err.Error())
+		} else {
+			recordCdrs = append(recordCdrs, storedCdr)
+		}
+	}
+	return recordCdrs, nil
+}
+
+func (self *FwvRecordsProcessor) recordPassesCfgFilter(record, configKey string) bool {
+	filterPasses := true
+	for _, rsrFilter := range self.cdrcCfgs[configKey].CdrFilter {
+		if rsrFilter == nil { // Nil filter does not need to match anything
+			continue
+		}
+		if cfgFieldIdx, _ := strconv.Atoi(rsrFilter.Id); len(record) <= cfgFieldIdx {
+			fmt.Errorf("Ignoring record: %v - cannot compile filter %+v", record, rsrFilter)
+			return false
+		} else if !rsrFilter.FilterPasses(record[cfgFieldIdx:]) {
+			fmt.Printf("Record content to test: %s\n", record[cfgFieldIdx:])
+			filterPasses = false
+			break
+		}
+	}
+	return filterPasses
+}
+
+// Converts a record (header or normal) to StoredCdr
+func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cfgKey string) (*engine.StoredCdr, error) {
+	var err error
+	var lazyHttpFields []*config.CfgCdrField
+	var cfgFields []*config.CfgCdrField
+	var duMultiplyFactor float64
+	var storedCdr *engine.StoredCdr
+	if self.headerCdr != nil { // Clone the header CDR so we can use it as base to future processing (inherit fields defined there)
+		storedCdr = self.headerCdr.Clone()
+	} else {
+		storedCdr = &engine.StoredCdr{CdrHost: "0.0.0.0", ExtraFields: make(map[string]string), Cost: -1}
+	}
+	if cfgKey == "*header" {
+		cfgFields = self.dfltCfg.HeaderFields
+		storedCdr.CdrSource = self.dfltCfg.CdrSourceId
+		duMultiplyFactor = self.dfltCfg.DataUsageMultiplyFactor
+	} else {
+		cfgFields = self.cdrcCfgs[cfgKey].ContentFields
+		storedCdr.CdrSource = self.cdrcCfgs[cfgKey].CdrSourceId
+		duMultiplyFactor = self.cdrcCfgs[cfgKey].DataUsageMultiplyFactor
+	}
+	for _, cdrFldCfg := range cfgFields {
+		var fieldVal string
+		switch cdrFldCfg.Type {
+		case utils.CDRFIELD:
+			for _, cfgFieldRSR := range cdrFldCfg.Value {
+				if cfgFieldRSR.IsStatic() {
+					fieldVal += cfgFieldRSR.ParseValue("")
+				} else { // Dynamic value extracted using index
+					if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx {
+						return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cdrFldCfg.Tag)
+					} else {
+						fieldVal += cfgFieldRSR.ParseValue(fwvValue(record, cfgFieldIdx, cdrFldCfg.Width, cdrFldCfg.Padding))
+					}
+				}
+			}
+		case utils.HTTP_POST:
+			lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of storedCdr to http server
+		default:
+			//return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type)
+			continue // Don't do anything for unsupported fields
+		}
+		if err := populateStoredCdrField(storedCdr, cdrFldCfg.CdrFieldId, fieldVal); err != nil {
+			return nil, err
+		}
+	}
+	if storedCdr.CgrId == "" && storedCdr.AccId != "" && cfgKey != "*header" {
+		storedCdr.CgrId = utils.Sha1(storedCdr.AccId, storedCdr.SetupTime.String())
+	}
+	if storedCdr.TOR == utils.DATA && duMultiplyFactor != 0 {
+		storedCdr.Usage = time.Duration(float64(storedCdr.Usage.Nanoseconds()) * duMultiplyFactor)
+	}
+	for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields
+		var outValByte []byte
+		var fieldVal, httpAddr string
+		for _, rsrFld := range httpFieldCfg.Value {
+			httpAddr += rsrFld.ParseValue("")
+		}
+		if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, storedCdr); err != nil && httpFieldCfg.Mandatory {
+			return nil, err
+		} else {
+			fieldVal = string(outValByte)
+			if len(fieldVal) == 0 && httpFieldCfg.Mandatory {
+				return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag)
+			}
+			if err := populateStoredCdrField(storedCdr, httpFieldCfg.CdrFieldId, fieldVal); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return storedCdr, nil
+}
+
+func (self *FwvRecordsProcessor) processHeader() error {
+	buf := make([]byte, self.lineLen)
+	if nRead, err := self.file.Read(buf); err != nil {
+		return err
+	} else if nRead != len(buf) {
+		return fmt.Errorf("In header, line len: %d, have read: %d", self.lineLen, nRead)
+	}
+	var err error
+	if self.headerCdr, err = self.recordToStoredCdr(string(buf), "*header"); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (self *FwvRecordsProcessor) processTrailer() error {
+	buf := make([]byte, self.lineLen)
+	if nRead, err := self.file.ReadAt(buf, self.trailerOffset); err != nil {
+		return err
+	} else if nRead != len(buf) {
+		return fmt.Errorf("In trailer, line len: %d, have read: %d", self.lineLen, nRead)
+	}
+	//engine.Logger.Debug(fmt.Sprintf("Have read trailer: <%q>", string(buf)))
+	return nil
+}

cdrc/fwv_local_test.go

@@ -0,0 +1,150 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"io/ioutil"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"os"
+	"path"
+	"testing"
+	"time"
+)
+
+var fwvCfgPath string
+var fwvCfg *config.CGRConfig
+var fwvRpc *rpc.Client
+var fwvCdrcCfg *config.CdrcConfig
+
+var FW_CDR_FILE1 = `HDR0001DDB     ABC                                     Some Connect A.B.                       DDB-Some-10022-20120711-309.CDR         00030920120711100255                                                                    
+CDR0000010  0 20120708181506000123451234         0040123123120                  004                                            000018009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009
+CDR0000020  0 20120708190945000123451234         0040123123120                  004                                            000016009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009
+CDR0000030  0 20120708191009000123451234         0040123123120                  004                                            000020009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009
+CDR0000040  0 20120708231043000123451234         0040123123120                  004                                            000011009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009
+CDR0000050  0 20120709122216000123451235         004212                         004                                            000217009980010001ISDN  ABC   10Buiten uw regio                         HMR 00000000190000000000
+CDR0000060  0 20120709130542000123451236         0012323453                     004                                            000019009980010001ISDN  ABC   35Sterdiensten                            AP  00000000190000000000
+CDR0000070  0 20120709140032000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000080  0 20120709140142000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000090  0 20120709150305000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000100  0 20120709150414000123451237         0040012323453100               001                                            000057009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000110  0 20120709150531000123451237         0040012323453100               001                                            000059009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000120  0 20120709150635000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000130  0 20120709151756000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000140  0 20120709154549000123451237         0040012323453100               001                                            000052009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000150  0 20120709154701000123451237         0040012323453100               001                                            000121009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000160  0 20120709154842000123451237         0040012323453100               001                                            000055009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000170  0 20120709154956000123451237         0040012323453100               001                                            000115009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000180  0 20120709155131000123451237         0040012323453100               001                                            000059009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000190  0 20120709155236000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000200  0 20120709160309000123451237         0040012323453100               001                                            000100009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000210  0 20120709160415000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000220  0 20120709161739000123451237         0040012323453100               001                                            000058009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000230  0 20120709170356000123123459         0040123234531                  004                                            000012002760010001ISDN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000240  0 20120709181036000123123450         0012323453                     004                                            000042009980010001ISDN  ABC   05Binnen uw regio                         AP  00000010190000000010
+CDR0000250  0 20120709191245000123123458         0040123232350                  004                                            000012002760000001PSTN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000260  0 20120709202324000123123459         0040123234531                  004                                            000011002760010001ISDN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000270  0 20120709211756000123451237         0040012323453100               001                                            000051009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000280  0 20120709211852000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000290  0 20120709212904000123123458         0040123232350                  004                                            000012002760000001PSTN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000300  0 20120709073707000123123459         0040123234531                  004                                            000012002760010001ISDN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000310  0 20120709085451000123451237         0040012323453100               001                                            000744009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000320  0 20120709091756000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000330  0 20120710070434000123123458         0040123232350                  004                                            000012002760000001PSTN  276   10Buiten uw regio                         TB  00000009190000000009
+TRL0001DDB     ABC                                     Some Connect A.B.                       DDB-Some-10022-20120711-309.CDR         0003090000003300000030550000000001000000000100Y                                         
+`
+
+func TestFwvLclInitCfg(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	fwvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcfwv")
+	if fwvCfg, err = config.NewCGRConfigFromFolder(fwvCfgPath); err != nil {
+		t.Fatal("Got config error: ", err.Error())
+	}
+}
+
+// Creates cdr files and moves them into processing folder
+func TestFwvLclCreateCdrFiles(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if fwvCfg == nil {
+		t.Fatal("Empty default cdrc configuration")
+	}
+	fwvCdrcCfg = fwvCfg.CdrcProfiles["/tmp/cgr_fwv/cdrc/in"]["FWV1"]
+	if err := os.RemoveAll(fwvCdrcCfg.CdrInDir); err != nil {
+		t.Fatal("Error removing folder: ", fwvCdrcCfg.CdrInDir, err)
+	}
+	if err := os.MkdirAll(fwvCdrcCfg.CdrInDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", fwvCdrcCfg.CdrInDir, err)
+	}
+	if err := os.RemoveAll(fwvCdrcCfg.CdrOutDir); err != nil {
+		t.Fatal("Error removing folder: ", fwvCdrcCfg.CdrOutDir, err)
+	}
+	if err := os.MkdirAll(fwvCdrcCfg.CdrOutDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", fwvCdrcCfg.CdrOutDir, err)
+	}
+}
+
+func TestFwvLclStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if _, err := engine.StopStartEngine(fwvCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestFwvLclRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	fwvRpc, err = jsonrpc.Dial("tcp", fwvCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+func TestFwvLclProcessFiles(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	fileName := "test1.fwv"
+	if err := ioutil.WriteFile(path.Join("/tmp", fileName), []byte(FW_CDR_FILE1), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	if err := os.Rename(path.Join("/tmp", fileName), path.Join(fwvCdrcCfg.CdrInDir, fileName)); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Duration(1) * time.Second)
+	filesInDir, _ := ioutil.ReadDir(fwvCdrcCfg.CdrInDir)
+	if len(filesInDir) != 0 {
+		t.Errorf("Files in cdrcInDir: %d", len(filesInDir))
+	}
+	filesOutDir, _ := ioutil.ReadDir(fwvCdrcCfg.CdrOutDir)
+	if len(filesOutDir) != 1 {
+		t.Errorf("In CdrcOutDir, expecting 1 files, got: %d", len(filesOutDir))
+	}
+}

cdrc/fwv_test.go

@@ -0,0 +1,55 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/utils"
+	"testing"
+)
+
+func TestFwvValue(t *testing.T) {
+	cdrLine := "CDR0000010  0 20120708181506000123451234         0040123123120                  004                                            000018009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009"
+	if val := fwvValue(cdrLine, 30, 19, "right"); val != "0123451234" {
+		t.Errorf("Received: <%s>", val)
+	}
+	if val := fwvValue(cdrLine, 14, 16, "right"); val != "2012070818150600" { // SetupTime
+		t.Errorf("Received: <%s>", val)
+	}
+	if val := fwvValue(cdrLine, 127, 8, "right"); val != "00001800" { // Usage
+		t.Errorf("Received: <%s>", val)
+	}
+	cdrLine = "HDR0001DDB     ABC                                     Some Connect A.B.                       DDB-Some-10022-20120711-309.CDR         00030920120711100255                                                "
+	if val := fwvValue(cdrLine, 135, 6, "zeroleft"); val != "309" {
+		t.Errorf("Received: <%s>", val)
+	}
+
+}
+
+func TestFwvRecordPassesCfgFilter(t *testing.T) {
+	//record, configKey string) bool {
+	cgrConfig, _ := config.NewDefaultCGRConfig()
+	cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][utils.META_DEFAULT] // We don't really care that is for .csv since all we want to test are the filters
+	cdrcConfig.CdrFilter = utils.ParseRSRFieldsMustCompile(`~52:s/^0(\d{9})/+49${1}/(+49123123120)`, utils.INFIELD_SEP)
+	fwvRp := &FwvRecordsProcessor{cdrcCfgs: cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"]}
+	cdrLine := "CDR0000010  0 20120708181506000123451234         0040123123120                  004                                            000018009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009"
+	if passesFilter := fwvRp.recordPassesCfgFilter(cdrLine, utils.META_DEFAULT); !passesFilter {
+		t.Error("Not passes filter")
+	}
+}

cdre/cdrexporter.go

@@ -1,6 +1,6 @@
 /*
 Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -22,53 +22,49 @@ import (
 	"encoding/csv"
 	"encoding/json"
 	"fmt"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
 	"io"
 	"os"
 	"strconv"
-	"strings"
 	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
 )
 
 const (
-	COST_DETAILS          = "cost_details"
-	FILLER                = "filler"
-	CONSTANT              = "constant"
-	METATAG               = "metatag"
-	CONCATENATED_CDRFIELD = "concatenated_cdrfield"
-	COMBIMED              = "combimed"
-	DATETIME              = "datetime"
-	HTTP_POST             = "http_post"
-	META_EXPORTID         = "export_id"
-	META_TIMENOW          = "time_now"
-	META_FIRSTCDRATIME    = "first_cdr_atime"
-	META_LASTCDRATIME     = "last_cdr_atime"
-	META_NRCDRS           = "cdrs_number"
-	META_DURCDRS          = "cdrs_duration"
-	META_SMSUSAGE         = "sms_usage"
-	META_DATAUSAGE        = "data_usage"
-	META_COSTCDRS         = "cdrs_cost"
-	META_MASKDESTINATION  = "mask_destination"
-	META_FORMATCOST       = "format_cost"
+	COST_DETAILS         = "cost_details"
+	DATETIME             = "datetime"
+	META_EXPORTID        = "*export_id"
+	META_TIMENOW         = "*time_now"
+	META_FIRSTCDRATIME   = "*first_cdr_atime"
+	META_LASTCDRATIME    = "*last_cdr_atime"
+	META_NRCDRS          = "*cdrs_number"
+	META_DURCDRS         = "*cdrs_duration"
+	META_SMSUSAGE        = "*sms_usage"
+	META_GENERICUSAGE    = "*generic_usage"
+	META_DATAUSAGE       = "*data_usage"
+	META_COSTCDRS        = "*cdrs_cost"
+	META_MASKDESTINATION = "*mask_destination"
+	META_FORMATCOST      = "*format_cost"
 )
 
 var err error
 
-func NewCdrExporter(cdrs []*utils.StoredCdr, logDb engine.LogStorage, exportTpl *config.CdreConfig, cdrFormat string, fieldSeparator rune, exportId string,
-	dataUsageMultiplyFactor, costMultiplyFactor float64, costShiftDigits, roundDecimals, cgrPrecision int, maskDestId string, maskLen int, httpSkipTlsCheck bool) (*CdrExporter, error) {
+func NewCdrExporter(cdrs []*engine.StoredCdr, cdrDb engine.CdrStorage, exportTpl *config.CdreConfig, cdrFormat string, fieldSeparator rune, exportId string,
+	dataUsageMultiplyFactor, smsUsageMultiplyFactor, genericUsageMultiplyFactor, costMultiplyFactor float64, costShiftDigits, roundDecimals, cgrPrecision int, maskDestId string, maskLen int, httpSkipTlsCheck bool) (*CdrExporter, error) {
 	if len(cdrs) == 0 { // Nothing to export
 		return nil, nil
 	}
 	cdre := &CdrExporter{
 		cdrs:                    cdrs,
-		logDb:                   logDb,
+		cdrDb:                   cdrDb,
 		exportTemplate:          exportTpl,
 		cdrFormat:               cdrFormat,
 		fieldSeparator:          fieldSeparator,
 		exportId:                exportId,
 		dataUsageMultiplyFactor: dataUsageMultiplyFactor,
+		smsUsageMultiplyFactor:  smsUsageMultiplyFactor,
 		costMultiplyFactor:      costMultiplyFactor,
 		costShiftDigits:         costShiftDigits,
 		roundDecimals:           roundDecimals,
@@ -85,22 +81,25 @@ func NewCdrExporter(cdrs []*utils.StoredCdr, logDb engine.LogStorage, exportTpl
 }
 
 type CdrExporter struct {
-	cdrs                                         []*utils.StoredCdr
-	logDb                                        engine.LogStorage // Used to extract cost_details if these are requested
-	exportTemplate                               *config.CdreConfig
-	cdrFormat                                    string // csv, fwv
-	fieldSeparator                               rune
-	exportId                                     string // Unique identifier or this export
-	dataUsageMultiplyFactor, costMultiplyFactor  float64
-	costShiftDigits, roundDecimals, cgrPrecision int
-	maskDestId                                   string
-	maskLen                                      int
-	httpSkipTlsCheck                             bool
-	header, trailer                              []string   // Header and Trailer fields
-	content                                      [][]string // Rows of cdr fields
-	firstCdrATime, lastCdrATime                  time.Time
-	numberOfRecords                              int
-	totalDuration, totalDataUsage, totalSmsUsage time.Duration
+	cdrs           []*engine.StoredCdr
+	cdrDb          engine.CdrStorage // Used to extract cost_details if these are requested
+	exportTemplate *config.CdreConfig
+	cdrFormat      string // csv, fwv
+	fieldSeparator rune
+	exportId       string // Unique identifier or this export
+	dataUsageMultiplyFactor,
+	smsUsageMultiplyFactor, // Multiply the SMS usage (eg: some billing systems billing them as minutes)
+	genericUsageMultiplyFactor,
+	costMultiplyFactor float64
+	costShiftDigits, roundDecimals, cgrPrecision                    int
+	maskDestId                                                      string
+	maskLen                                                         int
+	httpSkipTlsCheck                                                bool
+	header, trailer                                                 []string   // Header and Trailer fields
+	content                                                         [][]string // Rows of cdr fields
+	firstCdrATime, lastCdrATime                                     time.Time
+	numberOfRecords                                                 int
+	totalDuration, totalDataUsage, totalSmsUsage, totalGenericUsage time.Duration
 
 	totalCost                       float64
 	firstExpOrderId, lastExpOrderId int64
@@ -111,7 +110,7 @@ type CdrExporter struct {
 // Return Json marshaled callCost attached to
 // Keep it separately so we test only this part in local tests
 func (cdre *CdrExporter) getCdrCostDetails(cgrId, runId string) (string, error) {
-	cc, err := cdre.logDb.GetCallCostLog(cgrId, "", runId)
+	cc, err := cdre.cdrDb.GetCallCostLog(cgrId, "", runId)
 	if err != nil {
 		return "", err
 	} else if cc == nil {
@@ -121,20 +120,25 @@ func (cdre *CdrExporter) getCdrCostDetails(cgrId, runId string) (string, error)
 	return string(ccJson), nil
 }
 
-func (cdre *CdrExporter) getCombimedCdrFieldVal(processedCdr *utils.StoredCdr, filterRule, fieldRule *utils.RSRField) (string, error) {
-	fltrPass, ftrPassValue := processedCdr.PassesFieldFilter(filterRule)
-	if !fltrPass {
-		return "", nil
-	}
-	for _, cdr := range cdre.cdrs {
-		if cdr.CgrId != processedCdr.CgrId {
-			continue // We only care about cdrs with same primary cdr behind
+func (cdre *CdrExporter) getCombimedCdrFieldVal(processedCdr *engine.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) {
+	var combinedVal string // Will result as combination of the field values, filters must match
+	for _, filterRule := range cfgCdrFld.FieldFilter {
+		fltrPass, ftrPassValue := processedCdr.PassesFieldFilter(filterRule)
+		if !fltrPass {
+			return "", nil
 		}
-		if cdr.FieldAsString(&utils.RSRField{Id: filterRule.Id}) == ftrPassValue {
-			return cdr.FieldAsString(fieldRule), nil
+		for _, cdr := range cdre.cdrs {
+			if cdr.CgrId != processedCdr.CgrId {
+				continue // We only care about cdrs with same primary cdr behind
+			}
+			if cdr.FieldAsString(&utils.RSRField{Id: filterRule.Id}) == ftrPassValue { // First CDR with filte
+				for _, rsrRule := range cfgCdrFld.Value {
+					combinedVal += cdr.FieldAsString(rsrRule)
+				}
+			}
 		}
 	}
-	return "", nil
+	return combinedVal, nil
 }
 
 // Check if the destination should be masked in output
@@ -145,17 +149,20 @@ func (cdre *CdrExporter) maskedDestination(destination string) bool {
 	return false
 }
 
-func (cdre *CdrExporter) getDateTimeFieldVal(cdr *utils.StoredCdr, fltrRl, fieldRl *utils.RSRField, layout string) (string, error) {
-	if fieldRl == nil {
+func (cdre *CdrExporter) getDateTimeFieldVal(cdr *engine.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) {
+	if len(cfgCdrFld.Value) == 0 {
 		return "", nil
 	}
-	if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass {
-		return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl)
+	for _, fltrRl := range cfgCdrFld.FieldFilter {
+		if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass {
+			return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl)
+		}
 	}
+	layout := cfgCdrFld.Layout
 	if len(layout) == 0 {
 		layout = time.RFC3339
 	}
-	if dtFld, err := utils.ParseTimeDetectLayout(cdr.FieldAsString(fieldRl)); err != nil {
+	if dtFld, err := utils.ParseTimeDetectLayout(cdr.FieldAsString(cfgCdrFld.Value[0])); err != nil { // Only one rule makes sense here
 		return "", err
 	} else {
 		return dtFld.Format(layout), nil
@@ -163,39 +170,45 @@ func (cdre *CdrExporter) getDateTimeFieldVal(cdr *utils.StoredCdr, fltrRl, field
 }
 
 // Extracts the value specified by cfgHdr out of cdr
-func (cdre *CdrExporter) cdrFieldValue(cdr *utils.StoredCdr, fltrRl, rsrFld *utils.RSRField, layout string) (string, error) {
-	if rsrFld == nil {
-		return "", nil
-	}
-	if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass {
-		return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl)
+func (cdre *CdrExporter) cdrFieldValue(cdr *engine.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) {
+	for _, fltrRl := range cfgCdrFld.FieldFilter {
+		if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass {
+			return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl)
+		}
 	}
+	layout := cfgCdrFld.Layout
 	if len(layout) == 0 {
 		layout = time.RFC3339
 	}
-	var cdrVal string
-	switch rsrFld.Id {
-	case COST_DETAILS: // Special case when we need to further extract cost_details out of logDb
-		if cdrVal, err = cdre.getCdrCostDetails(cdr.CgrId, cdr.MediationRunId); err != nil {
-			return "", err
+	var retVal string // Concatenate the resulting values
+	for _, rsrFld := range cfgCdrFld.Value {
+		var cdrVal string
+		switch rsrFld.Id {
+		case COST_DETAILS: // Special case when we need to further extract cost_details out of logDb
+			if cdr.ExtraFields[COST_DETAILS], err = cdre.getCdrCostDetails(cdr.CgrId, cdr.MediationRunId); err != nil {
+				return "", err
+			} else {
+				cdrVal = cdr.FieldAsString(rsrFld)
+			}
+		case utils.COST:
+			cdrVal = cdr.FormatCost(cdre.costShiftDigits, cdre.roundDecimals)
+		case utils.USAGE:
+			cdrVal = cdr.FormatUsage(layout)
+		case utils.SETUP_TIME:
+			cdrVal = cdr.SetupTime.Format(layout)
+		case utils.ANSWER_TIME: // Format time based on layout
+			cdrVal = cdr.AnswerTime.Format(layout)
+		case utils.DESTINATION:
+			cdrVal = cdr.FieldAsString(rsrFld)
+			if cdre.maskLen != -1 && cdre.maskedDestination(cdrVal) {
+				cdrVal = MaskDestination(cdrVal, cdre.maskLen)
+			}
+		default:
+			cdrVal = cdr.FieldAsString(rsrFld)
 		}
-	case utils.COST:
-		cdrVal = cdr.FormatCost(cdre.costShiftDigits, cdre.roundDecimals)
-	case utils.USAGE:
-		cdrVal = cdr.FormatUsage(layout)
-	case utils.SETUP_TIME:
-		cdrVal = cdr.SetupTime.Format(layout)
-	case utils.ANSWER_TIME: // Format time based on layout
-		cdrVal = cdr.AnswerTime.Format(layout)
-	case utils.DESTINATION:
-		cdrVal = cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION})
-		if cdre.maskLen != -1 && cdre.maskedDestination(cdrVal) {
-			cdrVal = MaskDestination(cdrVal, cdre.maskLen)
-		}
-	default:
-		cdrVal = cdr.FieldAsString(rsrFld)
+		retVal += cdrVal
 	}
-	return rsrFld.ParseValue(cdrVal), nil
+	return retVal, nil
 }
 
 // Handle various meta functions used in header/trailer
@@ -212,16 +225,16 @@ func (cdre *CdrExporter) metaHandler(tag, arg string) (string, error) {
 	case META_NRCDRS:
 		return strconv.Itoa(cdre.numberOfRecords), nil
 	case META_DURCDRS:
-		//return strconv.FormatFloat(cdre.totalDuration.Seconds(), 'f', -1, 64), nil
-		emulatedCdr := &utils.StoredCdr{TOR: utils.VOICE, Usage: cdre.totalDuration}
+		emulatedCdr := &engine.StoredCdr{TOR: utils.VOICE, Usage: cdre.totalDuration}
 		return emulatedCdr.FormatUsage(arg), nil
 	case META_SMSUSAGE:
-		//return strconv.FormatFloat(cdre.totalDuration.Seconds(), 'f', -1, 64), nil
-		emulatedCdr := &utils.StoredCdr{TOR: utils.SMS, Usage: cdre.totalSmsUsage}
+		emulatedCdr := &engine.StoredCdr{TOR: utils.SMS, Usage: cdre.totalSmsUsage}
+		return emulatedCdr.FormatUsage(arg), nil
+	case META_GENERICUSAGE:
+		emulatedCdr := &engine.StoredCdr{TOR: utils.GENERIC, Usage: cdre.totalGenericUsage}
 		return emulatedCdr.FormatUsage(arg), nil
 	case META_DATAUSAGE:
-		//return strconv.FormatFloat(cdre.totalDuration.Seconds(), 'f', -1, 64), nil
-		emulatedCdr := &utils.StoredCdr{TOR: utils.DATA, Usage: cdre.totalDataUsage}
+		emulatedCdr := &engine.StoredCdr{TOR: utils.DATA, Usage: cdre.totalDataUsage}
 		return emulatedCdr.FormatUsage(arg), nil
 	case META_COSTCDRS:
 		return strconv.FormatFloat(utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE), 'f', -1, 64), nil
@@ -233,7 +246,6 @@ func (cdre *CdrExporter) metaHandler(tag, arg string) (string, error) {
 	default:
 		return "", fmt.Errorf("Unsupported METATAG: %s", tag)
 	}
-	return "", nil
 }
 
 // Compose and cache the header
@@ -241,23 +253,23 @@ func (cdre *CdrExporter) composeHeader() error {
 	for _, cfgFld := range cdre.exportTemplate.HeaderFields {
 		var outVal string
 		switch cfgFld.Type {
-		case FILLER:
-			outVal = cfgFld.Value
+		case utils.FILLER:
+			outVal = cfgFld.Value.Id()
 			cfgFld.Padding = "right"
-		case CONSTANT:
-			outVal = cfgFld.Value
-		case METATAG:
-			outVal, err = cdre.metaHandler(cfgFld.Value, cfgFld.Layout)
+		case utils.CONSTANT:
+			outVal = cfgFld.Value.Id()
+		case utils.METATAG:
+			outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cfgFld.Layout)
 		default:
 			return fmt.Errorf("Unsupported field type: %s", cfgFld.Type)
 		}
 		if err != nil {
-			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR header, field %s, error: %s", cfgFld.Name, err.Error()))
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR header, field %s, error: %s", cfgFld.Tag, err.Error()))
 			return err
 		}
 		fmtOut := outVal
 		if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
-			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR header, field %s, error: %s", cfgFld.Name, err.Error()))
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR header, field %s, error: %s", cfgFld.Tag, err.Error()))
 			return err
 		}
 		cdre.header = append(cdre.header, fmtOut)
@@ -270,23 +282,23 @@ func (cdre *CdrExporter) composeTrailer() error {
 	for _, cfgFld := range cdre.exportTemplate.TrailerFields {
 		var outVal string
 		switch cfgFld.Type {
-		case FILLER:
-			outVal = cfgFld.Value
+		case utils.FILLER:
+			outVal = cfgFld.Value.Id()
 			cfgFld.Padding = "right"
-		case CONSTANT:
-			outVal = cfgFld.Value
-		case METATAG:
-			outVal, err = cdre.metaHandler(cfgFld.Value, cfgFld.Layout)
+		case utils.CONSTANT:
+			outVal = cfgFld.Value.Id()
+		case utils.METATAG:
+			outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cfgFld.Layout)
 		default:
 			return fmt.Errorf("Unsupported field type: %s", cfgFld.Type)
 		}
 		if err != nil {
-			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, field: %s, error: %s", cfgFld.Name, err.Error()))
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, field: %s, error: %s", cfgFld.Tag, err.Error()))
 			return err
 		}
 		fmtOut := outVal
 		if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
-			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, field: %s, error: %s", cfgFld.Name, err.Error()))
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, field: %s, error: %s", cfgFld.Tag, err.Error()))
 			return err
 		}
 		cdre.trailer = append(cdre.trailer, fmtOut)
@@ -295,12 +307,17 @@ func (cdre *CdrExporter) composeTrailer() error {
 }
 
 // Write individual cdr into content buffer, build stats
-func (cdre *CdrExporter) processCdr(cdr *utils.StoredCdr) error {
+func (cdre *CdrExporter) processCdr(cdr *engine.StoredCdr) error {
 	if cdr == nil || len(cdr.CgrId) == 0 { // We do not export empty CDRs
 		return nil
 	}
+	// Cost multiply
 	if cdre.dataUsageMultiplyFactor != 0.0 && cdr.TOR == utils.DATA {
 		cdr.UsageMultiply(cdre.dataUsageMultiplyFactor, cdre.cgrPrecision)
+	} else if cdre.smsUsageMultiplyFactor != 0 && cdr.TOR == utils.SMS {
+		cdr.UsageMultiply(cdre.smsUsageMultiplyFactor, cdre.cgrPrecision)
+	} else if cdre.genericUsageMultiplyFactor != 0 && cdr.TOR == utils.GENERIC {
+		cdr.UsageMultiply(cdre.genericUsageMultiplyFactor, cdre.cgrPrecision)
 	}
 	if cdre.costMultiplyFactor != 0.0 {
 		cdr.CostMultiply(cdre.costMultiplyFactor, cdre.cgrPrecision)
@@ -310,38 +327,33 @@ func (cdre *CdrExporter) processCdr(cdr *utils.StoredCdr) error {
 	for idx, cfgFld := range cdre.exportTemplate.ContentFields {
 		var outVal string
 		switch cfgFld.Type {
-		case FILLER:
-			outVal = cfgFld.Value
+		case utils.FILLER:
+			outVal = cfgFld.Value.Id()
 			cfgFld.Padding = "right"
-		case CONSTANT:
-			outVal = cfgFld.Value
+		case utils.CONSTANT:
+			outVal = cfgFld.Value.Id()
 		case utils.CDRFIELD:
-			outVal, err = cdre.cdrFieldValue(cdr, cfgFld.Filter, cfgFld.ValueAsRSRField(), cfgFld.Layout)
+			outVal, err = cdre.cdrFieldValue(cdr, cfgFld)
 		case DATETIME:
-			outVal, err = cdre.getDateTimeFieldVal(cdr, cfgFld.Filter, cfgFld.ValueAsRSRField(), cfgFld.Layout)
-		case HTTP_POST:
+			outVal, err = cdre.getDateTimeFieldVal(cdr, cfgFld)
+		case utils.HTTP_POST:
 			var outValByte []byte
-			if outValByte, err = utils.HttpJsonPost(cfgFld.Value, cdre.httpSkipTlsCheck, cdr); err == nil {
+			httpAddr := cfgFld.Value.Id()
+			if len(httpAddr) == 0 {
+				err = fmt.Errorf("Empty http address for field %s type %s", cfgFld.Tag, cfgFld.Type)
+			} else if outValByte, err = utils.HttpJsonPost(httpAddr, cdre.httpSkipTlsCheck, cdr); err == nil {
 				outVal = string(outValByte)
 				if len(outVal) == 0 && cfgFld.Mandatory {
-					err = fmt.Errorf("Empty result for http_post field: %s", cfgFld.Name)
+					err = fmt.Errorf("Empty result for http_post field: %s", cfgFld.Tag)
 				}
 			}
-		case COMBIMED:
-			outVal, err = cdre.getCombimedCdrFieldVal(cdr, cfgFld.Filter, cfgFld.ValueAsRSRField())
-		case CONCATENATED_CDRFIELD:
-			for _, fld := range strings.Split(cfgFld.Value, ",") {
-				if fldOut, err := cdre.cdrFieldValue(cdr, cfgFld.Filter, &utils.RSRField{Id: fld}, cfgFld.Layout); err != nil {
-					break // The error will be reported bellow
-				} else {
-					outVal += fldOut
-				}
-			}
-		case METATAG:
-			if cfgFld.Value == META_MASKDESTINATION {
-				outVal, err = cdre.metaHandler(cfgFld.Value, cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION}))
+		case utils.COMBIMED:
+			outVal, err = cdre.getCombimedCdrFieldVal(cdr, cfgFld)
+		case utils.METATAG:
+			if cfgFld.Value.Id() == META_MASKDESTINATION {
+				outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION}))
 			} else {
-				outVal, err = cdre.metaHandler(cfgFld.Value, cfgFld.Layout)
+				outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cfgFld.Layout)
 			}
 		}
 		if err != nil {
@@ -350,7 +362,7 @@ func (cdre *CdrExporter) processCdr(cdr *utils.StoredCdr) error {
 		}
 		fmtOut := outVal
 		if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
-			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Name, outVal, err.Error()))
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Tag, outVal, err.Error()))
 			return err
 		}
 		cdrRow[idx] += fmtOut
@@ -374,7 +386,10 @@ func (cdre *CdrExporter) processCdr(cdr *utils.StoredCdr) error {
 	if cdr.TOR == utils.SMS { // Count usage for SMS
 		cdre.totalSmsUsage += cdr.Usage
 	}
-	if cdr.TOR == utils.DATA { // Count usage for SMS
+	if cdr.TOR == utils.GENERIC { // Count usage for GENERIC
+		cdre.totalGenericUsage += cdr.Usage
+	}
+	if cdr.TOR == utils.DATA { // Count usage for DATA
 		cdre.totalDataUsage += cdr.Usage
 	}
 	if cdr.Cost != -1 {
@@ -469,16 +484,16 @@ func (cdre *CdrExporter) WriteToFile(filePath string) error {
 	}
 	defer fileOut.Close()
 	switch cdre.cdrFormat {
-	case utils.CDRE_DRYRUN:
+	case utils.DRYRUN:
 		return nil
 	case utils.CDRE_FIXED_WIDTH:
 		if err := cdre.writeOut(fileOut); err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+			return utils.NewErrServerError(err)
 		}
 	case utils.CSV:
 		csvWriter := csv.NewWriter(fileOut)
 		if err := cdre.writeCsv(csvWriter); err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+			return utils.NewErrServerError(err)
 		}
 	}
 	return nil

cdre/cdrexporter_test.go

@@ -1,6 +1,6 @@
 /*
 Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -19,52 +19,55 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package cdre
 
 import (
+	"testing"
+	"time"
+
 	"github.com/cgrates/cgrates/config"
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
-	"testing"
-	"time"
 )
 
 func TestCdreGetCombimedCdrFieldVal(t *testing.T) {
-	logDb, _ := engine.NewMapStorage()
 	cfg, _ := config.NewDefaultCGRConfig()
-	cdrs := []*utils.StoredCdr{
-		&utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
-			ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 			Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
 			Usage: time.Duration(10) * time.Second, MediationRunId: "RUN_RTL", Cost: 1.01},
-		&utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf2", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
-			ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf2", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 			Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
 			Usage: time.Duration(10) * time.Second, MediationRunId: "CUSTOMER1", Cost: 2.01},
-		&utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
-			ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 			Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
 			Usage: time.Duration(10) * time.Second, MediationRunId: "CUSTOMER1", Cost: 3.01},
-		&utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
-			ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 			Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
 			Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 4.01},
-		&utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
-			ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 			Category: "call", Account: "1000", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
 			Usage: time.Duration(10) * time.Second, MediationRunId: "RETAIL1", Cost: 5.01},
 	}
-
-	cdre, err := NewCdrExporter(cdrs, logDb, cfg.CdreDefaultInstance, cfg.CdreDefaultInstance.CdrFormat, cfg.CdreDefaultInstance.FieldSeparator,
-		"firstexport", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", 0, cfg.HttpSkipTlsVerify)
+	cdre, err := NewCdrExporter(cdrs, nil, cfg.CdreProfiles["*default"], cfg.CdreProfiles["*default"].CdrFormat, cfg.CdreProfiles["*default"].FieldSeparator,
+		"firstexport", 0.0, 0.0, 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", 0, cfg.HttpSkipTlsVerify)
 	if err != nil {
 		t.Error("Unexpected error received: ", err)
 	}
-	fltrRule, _ := utils.NewRSRField("~mediation_runid:s/default/RUN_RTL/")
-	if costVal, err := cdre.getCombimedCdrFieldVal(cdrs[3], fltrRule, &utils.RSRField{Id: "cost"}); err != nil {
+	fltrRule, _ := utils.ParseRSRFields("~mediation_runid:s/default/RUN_RTL/", utils.INFIELD_SEP)
+	val, _ := utils.ParseRSRFields("cost", utils.INFIELD_SEP)
+	cfgCdrFld := &config.CfgCdrField{Tag: "cost", Type: "cdrfield", CdrFieldId: "cost", Value: val, FieldFilter: fltrRule}
+	if costVal, err := cdre.getCombimedCdrFieldVal(cdrs[3], cfgCdrFld); err != nil {
 		t.Error(err)
 	} else if costVal != "1.01" {
 		t.Error("Expecting: 1.01, received: ", costVal)
 	}
-	fltrRule, _ = utils.NewRSRField("~mediation_runid:s/default/RETAIL1/")
-	if acntVal, err := cdre.getCombimedCdrFieldVal(cdrs[3], fltrRule, &utils.RSRField{Id: "account"}); err != nil {
+	fltrRule, _ = utils.ParseRSRFields("~mediation_runid:s/default/RETAIL1/", utils.INFIELD_SEP)
+	val, _ = utils.ParseRSRFields("account", utils.INFIELD_SEP)
+	cfgCdrFld = &config.CfgCdrField{Tag: "account", Type: "cdrfield", CdrFieldId: "account", Value: val, FieldFilter: fltrRule}
+	if acntVal, err := cdre.getCombimedCdrFieldVal(cdrs[3], cfgCdrFld); err != nil {
 		t.Error(err)
 	} else if acntVal != "1000" {
 		t.Error("Expecting: 1000, received: ", acntVal)
@@ -73,41 +76,49 @@ func TestCdreGetCombimedCdrFieldVal(t *testing.T) {
 
 func TestGetDateTimeFieldVal(t *testing.T) {
 	cdreTst := new(CdrExporter)
-	cdrTst := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
-		ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+	cdrTst := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
 		Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.01,
 		ExtraFields: map[string]string{"stop_time": "2014-06-11 19:19:00 +0000 UTC", "fieldextr2": "valextr2"}}
-	if cdrVal, err := cdreTst.getDateTimeFieldVal(cdrTst, nil, &utils.RSRField{Id: "stop_time"}, "2006-01-02 15:04:05"); err != nil {
+	val, _ := utils.ParseRSRFields("stop_time", utils.INFIELD_SEP)
+	layout := "2006-01-02 15:04:05"
+	cfgCdrFld := &config.CfgCdrField{Tag: "stop_time", Type: "cdrfield", CdrFieldId: "stop_time", Value: val, Layout: layout}
+	if cdrVal, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err != nil {
 		t.Error(err)
 	} else if cdrVal != "2014-06-11 19:19:00" {
 		t.Error("Expecting: 2014-06-11 19:19:00, got: ", cdrVal)
 	}
 	// Test filter
-	fltrRule, _ := utils.NewRSRField("~tenant:s/(.+)/itsyscom.com/")
-	if _, err := cdreTst.getDateTimeFieldVal(cdrTst, fltrRule, &utils.RSRField{Id: "stop_time"}, "2006-01-02 15:04:05"); err == nil {
+	fltr, _ := utils.ParseRSRFields("~tenant:s/(.+)/itsyscom.com/", utils.INFIELD_SEP)
+	cfgCdrFld = &config.CfgCdrField{Tag: "stop_time", Type: "cdrfield", CdrFieldId: "stop_time", Value: val, FieldFilter: fltr, Layout: layout}
+	if _, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err == nil {
 		t.Error(err)
 	}
+	val, _ = utils.ParseRSRFields("fieldextr2", utils.INFIELD_SEP)
+	cfgCdrFld = &config.CfgCdrField{Tag: "stop_time", Type: "cdrfield", CdrFieldId: "stop_time", Value: val, Layout: layout}
 	// Test time parse error
-	if _, err := cdreTst.getDateTimeFieldVal(cdrTst, nil, &utils.RSRField{Id: "fieldextr2"}, "2006-01-02 15:04:05"); err == nil {
+	if _, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err == nil {
 		t.Error("Should give error here, got none.")
 	}
 }
 
 func TestCdreCdrFieldValue(t *testing.T) {
 	cdre := new(CdrExporter)
-	cdr := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
-		ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+	cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
 		Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.01}
-	fltrRule, _ := utils.NewRSRField("~tenant:s/(.+)/cgrates.org/")
-	if val, err := cdre.cdrFieldValue(cdr, fltrRule, &utils.RSRField{Id: "destination"}, ""); err != nil {
+	val, _ := utils.ParseRSRFields("destination", utils.INFIELD_SEP)
+	cfgCdrFld := &config.CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: val}
+	if val, err := cdre.cdrFieldValue(cdr, cfgCdrFld); err != nil {
 		t.Error(err)
 	} else if val != cdr.Destination {
 		t.Errorf("Expecting: %s, received: %s", cdr.Destination, val)
 	}
-	fltrRule, _ = utils.NewRSRField("~tenant:s/(.+)/itsyscom.com/")
-	if _, err := cdre.cdrFieldValue(cdr, fltrRule, &utils.RSRField{Id: "destination"}, ""); err == nil {
+	fltr, _ := utils.ParseRSRFields("~tenant:s/(.+)/itsyscom.com/", utils.INFIELD_SEP)
+	cfgCdrFld = &config.CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: val, FieldFilter: fltr}
+	if _, err := cdre.cdrFieldValue(cdr, cfgCdrFld); err == nil {
 		t.Error("Failed to use filter")
 	}
 }

cdre/csv_test.go

@@ -1,6 +1,6 @@
 /*
 Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -21,25 +21,26 @@ package cdre
 import (
 	"bytes"
 	"encoding/csv"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
 )
 
 func TestCsvCdrWriter(t *testing.T) {
 	writer := &bytes.Buffer{}
 	cfg, _ := config.NewDefaultCGRConfig()
-	logDb, _ := engine.NewMapStorage()
-	storedCdr1 := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
-		ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+	storedCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
 		Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID,
 		ExtraFields: map[string]string{"extra1": "val_extra1", "extra2": "val_extra2", "extra3": "val_extra3"}, Cost: 1.01,
 	}
-	cdre, err := NewCdrExporter([]*utils.StoredCdr{storedCdr1}, logDb, cfg.CdreDefaultInstance, utils.CSV, ',', "firstexport", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", 0, cfg.HttpSkipTlsVerify)
+	cdre, err := NewCdrExporter([]*engine.StoredCdr{storedCdr1}, nil, cfg.CdreProfiles["*default"], utils.CSV, ',', "firstexport", 0.0, 0.0, 0.0, 0.0, 0, 4,
+		cfg.RoundingDecimals, "", 0, cfg.HttpSkipTlsVerify)
 	if err != nil {
 		t.Error("Unexpected error received: ", err)
 	}
@@ -47,7 +48,7 @@ func TestCsvCdrWriter(t *testing.T) {
 	if err := cdre.writeCsv(csvWriter); err != nil {
 		t.Error("Unexpected error: ", err)
 	}
-	expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6,default,*voice,dsafdsaf,rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07T08:42:25Z,2013-11-07T08:42:26Z,10000000000,1.0100`
+	expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6,*default,*voice,dsafdsaf,*rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07T08:42:25Z,2013-11-07T08:42:26Z,10,1.0100`
 	result := strings.TrimSpace(writer.String())
 	if result != expected {
 		t.Errorf("Expected: \n%s received: \n%s.", expected, result)
@@ -60,14 +61,13 @@ func TestCsvCdrWriter(t *testing.T) {
 func TestAlternativeFieldSeparator(t *testing.T) {
 	writer := &bytes.Buffer{}
 	cfg, _ := config.NewDefaultCGRConfig()
-	logDb, _ := engine.NewMapStorage()
-	storedCdr1 := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
-		ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+	storedCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
 		Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID,
 		ExtraFields: map[string]string{"extra1": "val_extra1", "extra2": "val_extra2", "extra3": "val_extra3"}, Cost: 1.01,
 	}
-	cdre, err := NewCdrExporter([]*utils.StoredCdr{storedCdr1}, logDb, cfg.CdreDefaultInstance, utils.CSV, '|', "firstexport", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", 0, cfg.HttpSkipTlsVerify)
+	cdre, err := NewCdrExporter([]*engine.StoredCdr{storedCdr1}, nil, cfg.CdreProfiles["*default"], utils.CSV, '|', "firstexport", 0.0, 0.0, 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", 0, cfg.HttpSkipTlsVerify)
 	if err != nil {
 		t.Error("Unexpected error received: ", err)
 	}
@@ -75,7 +75,7 @@ func TestAlternativeFieldSeparator(t *testing.T) {
 	if err := cdre.writeCsv(csvWriter); err != nil {
 		t.Error("Unexpected error: ", err)
 	}
-	expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6|default|*voice|dsafdsaf|rated|*out|cgrates.org|call|1001|1001|1002|2013-11-07T08:42:25Z|2013-11-07T08:42:26Z|10000000000|1.0100`
+	expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6|*default|*voice|dsafdsaf|*rated|*out|cgrates.org|call|1001|1001|1002|2013-11-07T08:42:25Z|2013-11-07T08:42:26Z|10|1.0100`
 	result := strings.TrimSpace(writer.String())
 	if result != expected {
 		t.Errorf("Expected: \n%s received: \n%s.", expected, result)

cdre/fixedwidth_test.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -20,88 +20,120 @@ package cdre
 
 import (
 	"bytes"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
 	"math"
 	"testing"
 	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
 )
 
-var hdrCfgFlds = []*config.CgrXmlCfgCdrField{
-	&config.CgrXmlCfgCdrField{Name: "TypeOfRecord", Type: CONSTANT, Value: "10", Width: 2},
-	&config.CgrXmlCfgCdrField{Name: "Filler1", Type: FILLER, Width: 3},
-	&config.CgrXmlCfgCdrField{Name: "DistributorCode", Type: CONSTANT, Value: "VOI", Width: 3},
-	&config.CgrXmlCfgCdrField{Name: "FileSeqNr", Type: METATAG, Value: "export_id", Width: 5, Strip: "right", Padding: "zeroleft"},
-	&config.CgrXmlCfgCdrField{Name: "LastCdr", Type: METATAG, Value: META_LASTCDRATIME, Width: 12, Layout: "020106150400"},
-	&config.CgrXmlCfgCdrField{Name: "FileCreationfTime", Type: METATAG, Value: "time_now", Width: 12, Layout: "020106150400"},
-	&config.CgrXmlCfgCdrField{Name: "FileVersion", Type: CONSTANT, Value: "01", Width: 2},
-	&config.CgrXmlCfgCdrField{Name: "Filler2", Type: FILLER, Width: 105},
+var hdrJsnCfgFlds = []*config.CdrFieldJsonCfg{
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TypeOfRecord"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("10"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler1"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(3)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("DistributorCode"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("VOI"), Width: utils.IntPointer(3)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FileSeqNr"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_EXPORTID),
+		Width: utils.IntPointer(5), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("zeroleft")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("LastCdr"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_LASTCDRATIME),
+		Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FileCreationfTime"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_TIMENOW),
+		Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FileVersion"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("01"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler2"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(105)},
 }
 
-var contentCfgFlds = []*config.CgrXmlCfgCdrField{
-	&config.CgrXmlCfgCdrField{Name: "TypeOfRecord", Type: CONSTANT, Value: "20", Width: 2},
-	&config.CgrXmlCfgCdrField{Name: "Account", Type: utils.CDRFIELD, Value: utils.ACCOUNT, Width: 12, Strip: "left", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "Subject", Type: utils.CDRFIELD, Value: utils.SUBJECT, Width: 5, Strip: "right", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "CLI", Type: utils.CDRFIELD, Value: "cli", Width: 15, Strip: "xright", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "Destination", Type: utils.CDRFIELD, Value: utils.DESTINATION, Width: 24, Strip: "xright", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "TOR", Type: CONSTANT, Value: "02", Width: 2},
-	&config.CgrXmlCfgCdrField{Name: "SubtypeTOR", Type: CONSTANT, Value: "11", Width: 4, Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "SetupTime", Type: utils.CDRFIELD, Value: utils.SETUP_TIME, Width: 12, Strip: "right", Padding: "right", Layout: "020106150400"},
-	&config.CgrXmlCfgCdrField{Name: "Duration", Type: utils.CDRFIELD, Value: utils.USAGE, Width: 6, Strip: "right", Padding: "right", Layout: utils.SECONDS},
-	&config.CgrXmlCfgCdrField{Name: "DataVolume", Type: FILLER, Width: 6},
-	&config.CgrXmlCfgCdrField{Name: "TaxCode", Type: CONSTANT, Value: "1", Width: 1},
-	&config.CgrXmlCfgCdrField{Name: "OperatorCode", Type: utils.CDRFIELD, Value: "opercode", Width: 2, Strip: "right", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "ProductId", Type: utils.CDRFIELD, Value: "productid", Width: 5, Strip: "right", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "NetworkId", Type: CONSTANT, Value: "3", Width: 1},
-	&config.CgrXmlCfgCdrField{Name: "CallId", Type: utils.CDRFIELD, Value: utils.ACCID, Width: 16, Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "Filler", Type: FILLER, Width: 8},
-	&config.CgrXmlCfgCdrField{Name: "Filler", Type: FILLER, Width: 8},
-	&config.CgrXmlCfgCdrField{Name: "TerminationCode", Type: CONCATENATED_CDRFIELD, Value: "operator,product", Width: 5, Strip: "right", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "Cost", Type: utils.CDRFIELD, Value: utils.COST, Width: 9, Padding: "zeroleft"},
-	&config.CgrXmlCfgCdrField{Name: "DestinationPrivacy", Type: METATAG, Value: META_MASKDESTINATION, Width: 1},
+var contentJsnCfgFlds = []*config.CdrFieldJsonCfg{
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TypeOfRecord"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("20"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Account"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.ACCOUNT), Width: utils.IntPointer(12),
+		Strip: utils.StringPointer("left"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Subject"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.SUBJECT), Width: utils.IntPointer(5),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("CLI"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("cli"), Width: utils.IntPointer(15),
+		Strip: utils.StringPointer("xright"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Destination"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.DESTINATION), Width: utils.IntPointer(24),
+		Strip: utils.StringPointer("xright"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TOR"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("02"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("SubtypeTOR"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("11"), Width: utils.IntPointer(4),
+		Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("SetupTime"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.SETUP_TIME), Width: utils.IntPointer(12),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right"), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Duration"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.USAGE), Width: utils.IntPointer(6),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right"), Layout: utils.StringPointer(utils.SECONDS)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("DataVolume"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(6)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TaxCode"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("1"), Width: utils.IntPointer(1)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("OperatorCode"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("opercode"), Width: utils.IntPointer(2),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("ProductId"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("productid"), Width: utils.IntPointer(5),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("NetworkId"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("3"), Width: utils.IntPointer(1)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("CallId"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.ACCID), Width: utils.IntPointer(16),
+		Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(8)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(8)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TerminationCode"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("operator;product"),
+		Width: utils.IntPointer(5), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Cost"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.COST), Width: utils.IntPointer(9),
+		Padding: utils.StringPointer("zeroleft")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("DestinationPrivacy"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_MASKDESTINATION),
+		Width: utils.IntPointer(1)},
 }
 
-var trailerCfgFlds = []*config.CgrXmlCfgCdrField{
-	&config.CgrXmlCfgCdrField{Name: "TypeOfRecord", Type: CONSTANT, Value: "90", Width: 2},
-	&config.CgrXmlCfgCdrField{Name: "Filler1", Type: FILLER, Width: 3},
-	&config.CgrXmlCfgCdrField{Name: "DistributorCode", Type: CONSTANT, Value: "VOI", Width: 3},
-	&config.CgrXmlCfgCdrField{Name: "FileSeqNr", Type: METATAG, Value: META_EXPORTID, Width: 5, Strip: "right", Padding: "zeroleft"},
-	&config.CgrXmlCfgCdrField{Name: "NumberOfRecords", Type: METATAG, Value: META_NRCDRS, Width: 6, Padding: "zeroleft"},
-	&config.CgrXmlCfgCdrField{Name: "CdrsDuration", Type: METATAG, Value: META_DURCDRS, Width: 8, Padding: "zeroleft", Layout: "seconds"},
-	&config.CgrXmlCfgCdrField{Name: "FirstCdrTime", Type: METATAG, Value: META_FIRSTCDRATIME, Width: 12, Layout: "020106150400"},
-	&config.CgrXmlCfgCdrField{Name: "LastCdrTime", Type: METATAG, Value: META_LASTCDRATIME, Width: 12, Layout: "020106150400"},
-	&config.CgrXmlCfgCdrField{Name: "Filler2", Type: FILLER, Width: 93},
+var trailerJsnCfgFlds = []*config.CdrFieldJsonCfg{
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TypeOfRecord"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("90"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler1"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(3)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("DistributorCode"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("VOI"), Width: utils.IntPointer(3)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FileSeqNr"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_EXPORTID), Width: utils.IntPointer(5),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("zeroleft")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("NumberOfRecords"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_NRCDRS),
+		Width: utils.IntPointer(6), Padding: utils.StringPointer("zeroleft")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("CdrsDuration"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_DURCDRS),
+		Width: utils.IntPointer(8), Padding: utils.StringPointer("zeroleft"), Layout: utils.StringPointer(utils.SECONDS)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FirstCdrTime"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_FIRSTCDRATIME),
+		Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("LastCdrTime"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_LASTCDRATIME),
+		Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler2"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(93)},
 }
 
+var hdrCfgFlds, contentCfgFlds, trailerCfgFlds []*config.CfgCdrField
+
 // Write one CDR and test it's results only for content buffer
 func TestWriteCdr(t *testing.T) {
+	var err error
 	wrBuf := &bytes.Buffer{}
-	logDb, _ := engine.NewMapStorage()
 	cfg, _ := config.NewDefaultCGRConfig()
-	fixedWidth := utils.CDRE_FIXED_WIDTH
-	exportTpl := &config.CgrXmlCdreCfg{
-		CdrFormat: &fixedWidth,
-		Header:    &config.CgrXmlCfgCdrHeader{Fields: hdrCfgFlds},
-		Content:   &config.CgrXmlCfgCdrContent{Fields: contentCfgFlds},
-		Trailer:   &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds},
+	if hdrCfgFlds, err = config.CfgCdrFieldsFromCdrFieldsJsonCfg(hdrJsnCfgFlds); err != nil {
+		t.Error(err)
 	}
-	cdr := &utils.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()),
+	if contentCfgFlds, err = config.CfgCdrFieldsFromCdrFieldsJsonCfg(contentJsnCfgFlds); err != nil {
+		t.Error(err)
+	}
+	if trailerCfgFlds, err = config.CfgCdrFieldsFromCdrFieldsJsonCfg(trailerJsnCfgFlds); err != nil {
+		t.Error(err)
+	}
+	cdreCfg := &config.CdreConfig{
+		CdrFormat:     utils.CDRE_FIXED_WIDTH,
+		HeaderFields:  hdrCfgFlds,
+		ContentFields: contentCfgFlds,
+		TrailerFields: trailerCfgFlds,
+	}
+	cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()),
 		TOR: utils.VOICE, OrderId: 1, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
-		ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 		Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
 		SetupTime:  time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC),
 		AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC),
 		Usage:      time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.34567,
 		ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
 	}
-	cdre, err := NewCdrExporter([]*utils.StoredCdr{cdr}, logDb, exportTpl.AsCdreConfig(), utils.CDRE_FIXED_WIDTH, ',', "fwv_1", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1, cfg.HttpSkipTlsVerify)
+	cdre, err := NewCdrExporter([]*engine.StoredCdr{cdr}, nil, cdreCfg, utils.CDRE_FIXED_WIDTH, ',', "fwv_1", 0.0, 0.0, 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1, cfg.HttpSkipTlsVerify)
 	if err != nil {
 		t.Error(err)
 	}
-	eHeader := "10   VOI0000007111308420024031415390001                                                                                                         \n"
+	eHeader := "10   VOIfwv_107111308420018011511340001                                                                                                         \n"
 	eContentOut := "201001        1001                1002                    0211  07111308420010          1       3dsafdsaf                             0002.34570\n"
-	eTrailer := "90   VOI0000000000100000010071113084260071113084200                                                                                             \n"
+	eTrailer := "90   VOIfwv_100000100000010071113084200071113084200                                                                                             \n"
 	if err := cdre.writeOut(wrBuf); err != nil {
 		t.Error(err)
 	}
@@ -137,33 +169,31 @@ func TestWriteCdr(t *testing.T) {
 
 func TestWriteCdrs(t *testing.T) {
 	wrBuf := &bytes.Buffer{}
-	logDb, _ := engine.NewMapStorage()
-	fixedWidth := utils.CDRE_FIXED_WIDTH
-	exportTpl := &config.CgrXmlCdreCfg{
-		CdrFormat: &fixedWidth,
-		Header:    &config.CgrXmlCfgCdrHeader{Fields: hdrCfgFlds},
-		Content:   &config.CgrXmlCfgCdrContent{Fields: contentCfgFlds},
-		Trailer:   &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds},
+	cdreCfg := &config.CdreConfig{
+		CdrFormat:     utils.CDRE_FIXED_WIDTH,
+		HeaderFields:  hdrCfgFlds,
+		ContentFields: contentCfgFlds,
+		TrailerFields: trailerCfgFlds,
 	}
-	cdr1 := &utils.StoredCdr{CgrId: utils.Sha1("aaa1", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()),
-		TOR: utils.VOICE, OrderId: 2, AccId: "aaa1", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
+	cdr1 := &engine.StoredCdr{CgrId: utils.Sha1("aaa1", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()),
+		TOR: utils.VOICE, OrderId: 2, AccId: "aaa1", CdrHost: "192.168.1.1", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
 		Category: "call", Account: "1001", Subject: "1001", Destination: "1010",
 		SetupTime:  time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC),
 		AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC),
 		Usage:      time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.25,
 		ExtraFields: map[string]string{"productnumber": "12341", "fieldextr2": "valextr2"},
 	}
-	cdr2 := &utils.StoredCdr{CgrId: utils.Sha1("aaa2", time.Date(2013, 11, 7, 7, 42, 20, 0, time.UTC).String()),
-		TOR: utils.VOICE, OrderId: 4, AccId: "aaa2", CdrHost: "192.168.1.2", ReqType: "prepaid", Direction: "*out", Tenant: "cgrates.org",
+	cdr2 := &engine.StoredCdr{CgrId: utils.Sha1("aaa2", time.Date(2013, 11, 7, 7, 42, 20, 0, time.UTC).String()),
+		TOR: utils.VOICE, OrderId: 4, AccId: "aaa2", CdrHost: "192.168.1.2", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org",
 		Category: "call", Account: "1002", Subject: "1002", Destination: "1011",
 		SetupTime:  time.Date(2013, 11, 7, 7, 42, 20, 0, time.UTC),
 		AnswerTime: time.Date(2013, 11, 7, 7, 42, 26, 0, time.UTC),
 		Usage:      time.Duration(5) * time.Minute, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.40001,
 		ExtraFields: map[string]string{"productnumber": "12342", "fieldextr2": "valextr2"},
 	}
-	cdr3 := &utils.StoredCdr{}
-	cdr4 := &utils.StoredCdr{CgrId: utils.Sha1("aaa3", time.Date(2013, 11, 7, 9, 42, 18, 0, time.UTC).String()),
-		TOR: utils.VOICE, OrderId: 3, AccId: "aaa4", CdrHost: "192.168.1.4", ReqType: "postpaid", Direction: "*out", Tenant: "cgrates.org",
+	cdr3 := &engine.StoredCdr{}
+	cdr4 := &engine.StoredCdr{CgrId: utils.Sha1("aaa3", time.Date(2013, 11, 7, 9, 42, 18, 0, time.UTC).String()),
+		TOR: utils.VOICE, OrderId: 3, AccId: "aaa4", CdrHost: "192.168.1.4", ReqType: utils.META_POSTPAID, Direction: "*out", Tenant: "cgrates.org",
 		Category: "call", Account: "1004", Subject: "1004", Destination: "1013",
 		SetupTime:  time.Date(2013, 11, 7, 9, 42, 18, 0, time.UTC),
 		AnswerTime: time.Date(2013, 11, 7, 9, 42, 26, 0, time.UTC),
@@ -171,8 +201,8 @@ func TestWriteCdrs(t *testing.T) {
 		ExtraFields: map[string]string{"productnumber": "12344", "fieldextr2": "valextr2"},
 	}
 	cfg, _ := config.NewDefaultCGRConfig()
-	cdre, err := NewCdrExporter([]*utils.StoredCdr{cdr1, cdr2, cdr3, cdr4}, logDb, exportTpl.AsCdreConfig(), utils.CDRE_FIXED_WIDTH, ',',
-		"fwv_1", 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1, cfg.HttpSkipTlsVerify)
+	cdre, err := NewCdrExporter([]*engine.StoredCdr{cdr1, cdr2, cdr3, cdr4}, nil, cdreCfg, utils.CDRE_FIXED_WIDTH, ',',
+		"fwv_1", 0.0, 0.0, 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1, cfg.HttpSkipTlsVerify)
 	if err != nil {
 		t.Error(err)
 	}

cdre/libcdre.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@ package cdre
 import (
 	"errors"
 	"fmt"
+
 	"github.com/cgrates/cgrates/utils"
 )
 

cdre/libcdre_test.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

cmd/cgr-console/cgr-console.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -24,14 +24,13 @@ import (
 	"fmt"
 	"io"
 	"log"
-	"net/rpc"
-	"net/rpc/jsonrpc"
 	"os"
 	"strings"
 
 	"github.com/cgrates/cgrates/console"
 	"github.com/cgrates/cgrates/utils"
 	"github.com/cgrates/liner"
+	"github.com/cgrates/rpcclient"
 )
 
 var (
@@ -40,7 +39,7 @@ var (
 	verbose      = flag.Bool("verbose", false, "Show extra info about command execution.")
 	server       = flag.String("server", "127.0.0.1:2012", "server address host:port")
 	rpc_encoding = flag.String("rpc_encoding", "json", "RPC encoding used <gob|json>")
-	client       *rpc.Client
+	client       *rpcclient.RpcClient
 )
 
 func executeCommand(command string) {
@@ -78,13 +77,20 @@ func executeCommand(command string) {
 	}
 	if cmd.RpcMethod() != "" {
 		res := cmd.RpcResult()
-		param := cmd.RpcParams()
+		param := cmd.RpcParams(false)
 		//log.Print(reflect.TypeOf(param))
 		switch param.(type) {
+		case *console.EmptyWrapper:
+			param = ""
 		case *console.StringWrapper:
 			param = param.(*console.StringWrapper).Item
+		case *console.StringSliceWrapper:
+			param = param.(*console.StringSliceWrapper).Items
+		case *console.StringMapWrapper:
+			param = param.(*console.StringMapWrapper).Items
 		}
 		//log.Printf("Param: %+v", param)
+
 		if rpcErr := client.Call(cmd.RpcMethod(), param, res); rpcErr != nil {
 			fmt.Println("Error executing command: " + rpcErr.Error())
 		} else {
@@ -102,18 +108,12 @@ func main() {
 		fmt.Println("CGRateS " + utils.VERSION)
 		return
 	}
-
 	var err error
-	if *rpc_encoding == "json" {
-		client, err = jsonrpc.Dial("tcp", *server)
-	} else {
-		client, err = rpc.Dial("tcp", *server)
-	}
+	client, err = rpcclient.NewRpcClient("tcp", *server, 3, 3, *rpc_encoding)
 	if err != nil {
 		flag.PrintDefaults()
 		log.Fatal("Could not connect to server " + *server)
 	}
-	defer client.Close()
 
 	if len(flag.Args()) != 0 {
 		executeCommand(strings.Join(flag.Args(), " "))
@@ -121,7 +121,7 @@ func main() {
 	}
 
 	fmt.Println("Welcome to CGRateS console!")
-	fmt.Println("Type `help` for a list of commands\n")
+	fmt.Print("Type `help` for a list of commands\n\n")
 
 	line := liner.NewLiner()
 	defer line.Close()

cmd/cgr-engine/cgr-engine.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -19,16 +19,18 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package main
 
 import (
-	"errors"
 	"flag"
 	"fmt"
 	"log"
 	"os"
-	//"runtime"
+	"runtime"
+	"runtime/pprof"
 	"strconv"
+	"sync"
 	"time"
 
-	"github.com/cgrates/cgrates/apier"
+	"github.com/cgrates/cgrates/apier/v1"
+	"github.com/cgrates/cgrates/apier/v2"
 	"github.com/cgrates/cgrates/balancer2go"
 	"github.com/cgrates/cgrates/cdrc"
 	"github.com/cgrates/cgrates/config"
@@ -49,86 +51,62 @@ const (
 	REDIS    = "redis"
 	SAME     = "same"
 	FS       = "freeswitch"
+	KAMAILIO = "kamailio"
 	OSIPS    = "opensips"
 )
 
 var (
-	cfgPath         = flag.String("config", "/etc/cgrates/cgrates.cfg", "Configuration file location.")
-	version         = flag.Bool("version", false, "Prints the application version.")
-	raterEnabled    = flag.Bool("rater", false, "Enforce starting of the rater daemon overwriting config")
-	schedEnabled    = flag.Bool("scheduler", false, "Enforce starting of the scheduler daemon .overwriting config")
-	cdrsEnabled     = flag.Bool("cdrs", false, "Enforce starting of the cdrs daemon overwriting config")
-	cdrcEnabled     = flag.Bool("cdrc", false, "Enforce starting of the cdrc service overwriting config")
-	mediatorEnabled = flag.Bool("mediator", false, "Enforce starting of the mediator service overwriting config")
-	pidFile         = flag.String("pid", "", "Write pid file")
-	bal             = balancer2go.NewBalancer()
-	exitChan        = make(chan bool)
-	server          = &engine.Server{}
-	scribeServer    history.Scribe
-	cdrServer       *engine.CDRS
-	cdrStats        *engine.Stats
-	sm              sessionmanager.SessionManager
-	medi            *engine.Mediator
-	cfg             *config.CGRConfig
-	err             error
+	cfgDir       = flag.String("config_dir", utils.CONFIG_DIR, "Configuration directory path.")
+	version      = flag.Bool("version", false, "Prints the application version.")
+	raterEnabled = flag.Bool("rater", false, "Enforce starting of the rater daemon overwriting config")
+	schedEnabled = flag.Bool("scheduler", false, "Enforce starting of the scheduler daemon .overwriting config")
+	cdrsEnabled  = flag.Bool("cdrs", false, "Enforce starting of the cdrs daemon overwriting config")
+	pidFile      = flag.String("pid", "", "Write pid file")
+	cpuprofile   = flag.String("cpuprofile", "", "write cpu profile to file")
+	singlecpu    = flag.Bool("singlecpu", false, "Run on single CPU core")
+	bal          = balancer2go.NewBalancer()
+	exitChan     = make(chan bool)
+	server       = &engine.Server{}
+	cdrServer    *engine.CdrServer
+	cdrStats     engine.StatsInterface
+	scribeServer history.Scribe
+	pubSubServer engine.PublisherSubscriber
+	userServer   engine.UserService
+	cfg          *config.CGRConfig
+	sms          []sessionmanager.SessionManager
+	smRpc        *v1.SessionManagerV1
+	err          error
 )
 
 func cacheData(ratingDb engine.RatingStorage, accountDb engine.AccountingStorage, doneChan chan struct{}) {
-	if err := ratingDb.CacheRating(nil, nil, nil, nil, nil); err != nil {
+	if err := ratingDb.CacheAll(); err != nil {
 		engine.Logger.Crit(fmt.Sprintf("Cache rating error: %s", err.Error()))
 		exitChan <- true
 		return
 	}
-	if err := accountDb.CacheAccounting(nil, nil, nil, nil); err != nil {
-		engine.Logger.Crit(fmt.Sprintf("Cache accounting error: %s", err.Error()))
-		exitChan <- true
-		return
-	}
 	close(doneChan)
 }
 
-func startMediator(responder *engine.Responder, loggerDb engine.LogStorage, cdrDb engine.CdrStorage, cacheChan, chanDone chan struct{}) {
-	var connector engine.Connector
-	if cfg.MediatorRater == utils.INTERNAL {
-		<-cacheChan // Cache needs to come up before we are ready
-		connector = responder
+// Fires up a cdrc instance
+func startCdrc(responder *engine.Responder, cdrsChan chan struct{}, cdrcCfgs map[string]*config.CdrcConfig, httpSkipTlsCheck bool, closeChan chan struct{}) {
+	var cdrsConn engine.Connector
+	var cdrcCfg *config.CdrcConfig
+	for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one
+		break
+	}
+	if cdrcCfg.Cdrs == utils.INTERNAL {
+		<-cdrsChan // Wait for CDRServer to come up before start processing
+		cdrsConn = responder
 	} else {
-		var client *rpcclient.RpcClient
-		var err error
-
-		for i := 0; i < cfg.MediatorReconnects; i++ {
-			client, err = rpcclient.NewRpcClient("tcp", cfg.MediatorRater, 0, cfg.MediatorReconnects, utils.GOB)
-			if err == nil { //Connected so no need to reiterate
-				break
-			}
-			time.Sleep(time.Duration(i+1) * time.Second)
-		}
+		conn, err := rpcclient.NewRpcClient("tcp", cdrcCfg.Cdrs, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
 		if err != nil {
-			engine.Logger.Crit(fmt.Sprintf("<Mediator> Could not connect to engine: %v", err))
+			engine.Logger.Crit(fmt.Sprintf("<CDRC> Could not connect to CDRS via RPC: %v", err))
 			exitChan <- true
 			return
 		}
-		connector = &engine.RPCClientConnector{Client: client}
+		cdrsConn = &engine.RPCClientConnector{Client: conn}
 	}
-	var err error
-	medi, err = engine.NewMediator(connector, loggerDb, cdrDb, cdrStats, cfg)
-	if err != nil {
-		engine.Logger.Crit(fmt.Sprintf("Mediator config parsing error: %v", err))
-		exitChan <- true
-		return
-	}
-	engine.Logger.Info("Registering Mediator RPC service.")
-	server.RpcRegister(&apier.MediatorV1{Medi: medi})
-
-	close(chanDone)
-}
-
-// Fires up a cdrc instance
-func startCdrc(cdrsChan chan struct{}, cdrsAddress, cdrType, cdrInDir, cdrOutDir, cdrSourceId string, runDelay time.Duration, csvSep string, cdrFields map[string][]*utils.RSRField) {
-	if cdrsAddress == utils.INTERNAL {
-		<-cdrsChan // Wait for CDRServer to come up before start processing
-	}
-	cdrc, err := cdrc.NewCdrc(cdrsAddress, cdrType, cdrInDir, cdrOutDir, cdrSourceId, runDelay, csvSep, cdrFields, cdrServer)
+	cdrc, err := cdrc.NewCdrc(cdrcCfgs, httpSkipTlsCheck, cdrsConn, closeChan)
 	if err != nil {
 		engine.Logger.Crit(fmt.Sprintf("Cdrc config parsing error: %s", err.Error()))
 		exitChan <- true
@@ -140,116 +118,222 @@ func startCdrc(cdrsChan chan struct{}, cdrsAddress, cdrType, cdrInDir, cdrOutDir
 	exitChan <- true // If run stopped, something is bad, stop the application
 }
 
-func startSessionManager(responder *engine.Responder, loggerDb engine.LogStorage, cacheChan chan struct{}) {
+func startSmFreeSWITCH(responder *engine.Responder, cdrDb engine.CdrStorage, cacheChan chan struct{}) {
 	var raterConn, cdrsConn engine.Connector
 	var client *rpcclient.RpcClient
-	if cfg.SMRater == utils.INTERNAL {
+	delay := utils.Fib()
+	if cfg.SmFsConfig.Rater == utils.INTERNAL {
 		<-cacheChan // Wait for the cache to init before start doing queries
 		raterConn = responder
 	} else {
 		var err error
-		for i := 0; i < cfg.SMReconnects; i++ {
-			client, err = rpcclient.NewRpcClient("tcp", cfg.SMRater, 0, cfg.SMReconnects, utils.GOB)
+		for i := 0; i < cfg.SmFsConfig.Reconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.SmFsConfig.Rater, cfg.ConnectAttempts, cfg.SmFsConfig.Reconnects, utils.GOB)
 			if err == nil { //Connected so no need to reiterate
 				break
 			}
-			time.Sleep(time.Duration(i+1) * time.Second)
+			time.Sleep(delay())
 		}
 		if err != nil {
-			engine.Logger.Crit(fmt.Sprintf("<SessionManager> Could not connect to engine: %v", err))
+			engine.Logger.Crit(fmt.Sprintf("<SM-FreeSWITCH> Could not connect to rater via RPC: %v", err))
 			exitChan <- true
+			return
 		}
 		raterConn = &engine.RPCClientConnector{Client: client}
 	}
-	if cfg.SMCdrS == cfg.SMRater {
+	if cfg.SmFsConfig.Cdrs == cfg.SmFsConfig.Rater {
 		cdrsConn = raterConn
-	} else if cfg.SMCdrS == utils.INTERNAL {
+	} else if cfg.SmFsConfig.Cdrs == utils.INTERNAL {
 		<-cacheChan // Wait for the cache to init before start doing queries
 		cdrsConn = responder
-	} else if len(cfg.SMCdrS) != 0 {
-		for i := 0; i < cfg.SMReconnects; i++ {
-			client, err = rpcclient.NewRpcClient("tcp", cfg.SMCdrS, 0, cfg.SMReconnects, utils.GOB)
+	} else if len(cfg.SmFsConfig.Cdrs) != 0 {
+		delay = utils.Fib()
+		for i := 0; i < cfg.SmFsConfig.Reconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.SmFsConfig.Cdrs, cfg.ConnectAttempts, cfg.SmFsConfig.Reconnects, utils.GOB)
 			if err == nil { //Connected so no need to reiterate
 				break
 			}
-			time.Sleep(time.Duration(i+1) * time.Second)
+			time.Sleep(delay())
 		}
 		if err != nil {
-			engine.Logger.Crit(fmt.Sprintf("<SM-OpenSIPS> Could not connect to CDRS via RPC: %v", err))
+			engine.Logger.Crit(fmt.Sprintf("<SM-FreeSWITCH> Could not connect to CDRS via RPC: %v", err))
 			exitChan <- true
+			return
 		}
 		cdrsConn = &engine.RPCClientConnector{Client: client}
 	}
-	switch cfg.SMSwitchType {
-	case FS:
-		dp, _ := time.ParseDuration(fmt.Sprintf("%vs", cfg.SMDebitInterval))
-		sm = sessionmanager.NewFSSessionManager(cfg, loggerDb, raterConn, cdrsConn, dp)
-	case OSIPS:
-		sm, _ = sessionmanager.NewOSipsSessionManager(cfg, raterConn, cdrsConn)
-	default:
-		engine.Logger.Err(fmt.Sprintf("<SessionManager> Unsupported session manger type: %s!", cfg.SMSwitchType))
-		exitChan <- true
-	}
+	sm := sessionmanager.NewFSSessionManager(cfg.SmFsConfig, raterConn, cdrsConn)
+	sms = append(sms, sm)
+	smRpc.SMs = append(smRpc.SMs, sm)
 	if err = sm.Connect(); err != nil {
 		engine.Logger.Err(fmt.Sprintf("<SessionManager> error: %s!", err))
 	}
 	exitChan <- true
 }
 
-func startCDRS(responder *engine.Responder, cdrDb engine.CdrStorage, mediChan, doneChan chan struct{}) {
-	if cfg.CDRSMediator == utils.INTERNAL {
-		<-mediChan // Deadlock if mediator not started
-		if medi == nil {
-			engine.Logger.Crit("<CDRS> Could not connect to mediator, exiting.")
+func startSmKamailio(responder *engine.Responder, cdrDb engine.CdrStorage, cacheChan chan struct{}) {
+	var raterConn, cdrsConn engine.Connector
+	var client *rpcclient.RpcClient
+	if cfg.SmKamConfig.Rater == utils.INTERNAL {
+		<-cacheChan // Wait for the cache to init before start doing queries
+		raterConn = responder
+	} else {
+		var err error
+		delay := utils.Fib()
+		for i := 0; i < cfg.SmKamConfig.Reconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.SmKamConfig.Rater, cfg.ConnectAttempts, cfg.SmKamConfig.Reconnects, utils.GOB)
+			if err == nil { //Connected so no need to reiterate
+				break
+			}
+			time.Sleep(delay())
+		}
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<SessionManager> Could not connect to rater: %v", err))
+			exitChan <- true
+		}
+		raterConn = &engine.RPCClientConnector{Client: client}
+	}
+	if cfg.SmKamConfig.Cdrs == cfg.SmKamConfig.Rater {
+		cdrsConn = raterConn
+	} else if cfg.SmKamConfig.Cdrs == utils.INTERNAL {
+		<-cacheChan // Wait for the cache to init before start doing queries
+		cdrsConn = responder
+	} else if len(cfg.SmKamConfig.Cdrs) != 0 {
+		delay := utils.Fib()
+		for i := 0; i < cfg.SmKamConfig.Reconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.SmKamConfig.Cdrs, cfg.ConnectAttempts, cfg.SmKamConfig.Reconnects, utils.GOB)
+			if err == nil { //Connected so no need to reiterate
+				break
+			}
+			time.Sleep(delay())
+		}
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<SM-Kamailio> Could not connect to CDRS via RPC: %v", err))
 			exitChan <- true
 			return
 		}
+		cdrsConn = &engine.RPCClientConnector{Client: client}
 	}
-	cdrServer = engine.NewCdrS(cdrDb, medi, cdrStats, cfg)
-	cdrServer.RegisterHanlersToServer(server)
-	engine.Logger.Info("Registering CDRS RPC service.")
-	server.RpcRegister(&apier.CDRSV1{CdrSrv: cdrServer})
-	responder.CdrSrv = cdrServer // Make the cdrserver available for internal communication
-	close(doneChan)
+	sm, _ := sessionmanager.NewKamailioSessionManager(cfg.SmKamConfig, raterConn, cdrsConn)
+	sms = append(sms, sm)
+	smRpc.SMs = append(smRpc.SMs, sm)
+	if err = sm.Connect(); err != nil {
+		engine.Logger.Err(fmt.Sprintf("<SessionManager> error: %s!", err))
+	}
+	exitChan <- true
 }
 
-func startHistoryServer(chanDone chan struct{}) {
-	if scribeServer, err = history.NewFileScribe(cfg.HistoryDir, cfg.HistorySaveInterval); err != nil {
-		engine.Logger.Crit(fmt.Sprintf("<HistoryServer> Could not start, error: %s", err.Error()))
-		exitChan <- true
-		return
+func startSmOpenSIPS(responder *engine.Responder, cdrDb engine.CdrStorage, cacheChan chan struct{}) {
+	var raterConn, cdrsConn engine.Connector
+	var client *rpcclient.RpcClient
+	if cfg.SmOsipsConfig.Rater == utils.INTERNAL {
+		<-cacheChan // Wait for the cache to init before start doing queries
+		raterConn = responder
+	} else {
+		var err error
+		delay := utils.Fib()
+		for i := 0; i < cfg.SmOsipsConfig.Reconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.SmOsipsConfig.Rater, cfg.ConnectAttempts, cfg.SmOsipsConfig.Reconnects, utils.GOB)
+			if err == nil { //Connected so no need to reiterate
+				break
+			}
+			time.Sleep(delay())
+		}
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<SessionManager> Could not connect to rater: %v", err))
+			exitChan <- true
+		}
+		raterConn = &engine.RPCClientConnector{Client: client}
 	}
-	server.RpcRegisterName("Scribe", scribeServer)
-	close(chanDone)
-}
-
-// chanStartServer will report when server is up, useful for internal requests
-func startHistoryAgent(chanServerStarted chan struct{}) {
-	if cfg.HistoryServer == utils.INTERNAL { // For internal requests, wait for server to come online before connecting
-		engine.Logger.Crit(fmt.Sprintf("<HistoryAgent> Connecting internally to HistoryServer"))
-		select {
-		case <-time.After(1 * time.Minute):
-			engine.Logger.Crit(fmt.Sprintf("<HistoryAgent> Timeout waiting for server to start."))
+	if cfg.SmOsipsConfig.Cdrs == cfg.SmOsipsConfig.Rater {
+		cdrsConn = raterConn
+	} else if cfg.SmOsipsConfig.Cdrs == utils.INTERNAL {
+		<-cacheChan // Wait for the cache to init before start doing queries
+		cdrsConn = responder
+	} else if len(cfg.SmOsipsConfig.Cdrs) != 0 {
+		delay := utils.Fib()
+		for i := 0; i < cfg.SmOsipsConfig.Reconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.SmOsipsConfig.Cdrs, cfg.ConnectAttempts, cfg.SmOsipsConfig.Reconnects, utils.GOB)
+			if err == nil { //Connected so no need to reiterate
+				break
+			}
+			time.Sleep(delay())
+		}
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<SM-OpenSIPS> Could not connect to CDRS via RPC: %v", err))
 			exitChan <- true
 			return
-		case <-chanServerStarted:
 		}
-		//<-chanServerStarted // If server is not enabled, will have deadlock here
-	} else { // Connect in iteration since there are chances of concurrency here
-		for i := 0; i < 3; i++ { //ToDo: Make it globally configurable
-			//engine.Logger.Crit(fmt.Sprintf("<HistoryAgent> Trying to connect, iteration: %d, time %s", i, time.Now()))
-			if scribeServer, err = history.NewProxyScribe(cfg.HistoryServer); err == nil {
-				break //Connected so no need to reiterate
-			} else if i == 2 && err != nil {
-				engine.Logger.Crit(fmt.Sprintf("<HistoryAgent> Could not connect to the server, error: %s", err.Error()))
+		cdrsConn = &engine.RPCClientConnector{Client: client}
+	}
+	sm, _ := sessionmanager.NewOSipsSessionManager(cfg.SmOsipsConfig, raterConn, cdrsConn)
+	sms = append(sms, sm)
+	smRpc.SMs = append(smRpc.SMs, sm)
+	if err := sm.Connect(); err != nil {
+		engine.Logger.Err(fmt.Sprintf("<SM-OpenSIPS> error: %s!", err))
+	}
+	exitChan <- true
+}
+
+func startCDRS(logDb engine.LogStorage, cdrDb engine.CdrStorage, responder *engine.Responder, responderReady, doneChan chan struct{}) {
+	var err error
+	var client *rpcclient.RpcClient
+	// Rater connection init
+	var raterConn engine.Connector
+	if cfg.CDRSRater == utils.INTERNAL {
+		<-responderReady // Wait for the cache to init before start doing queries
+		raterConn = responder
+	} else if len(cfg.CDRSRater) != 0 {
+		delay := utils.Fib()
+		for i := 0; i < cfg.CDRSReconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSRater, cfg.ConnectAttempts, cfg.CDRSReconnects, utils.GOB)
+			if err == nil { //Connected so no need to reiterate
+				break
+			}
+			time.Sleep(delay())
+		}
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<CDRS> Could not connect to rater: %s", err.Error()))
+			exitChan <- true
+			return
+		}
+		raterConn = &engine.RPCClientConnector{Client: client}
+	}
+	// Stats connection init
+	var statsConn engine.StatsInterface
+	if cfg.CDRSStats == utils.INTERNAL {
+		statsConn = cdrStats
+	} else if len(cfg.CDRSStats) != 0 {
+		if cfg.CDRSRater == cfg.CDRSStats {
+			statsConn = &engine.ProxyStats{Client: client}
+		} else {
+			delay := utils.Fib()
+			for i := 0; i < cfg.CDRSReconnects; i++ {
+				client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSStats, cfg.ConnectAttempts, cfg.CDRSReconnects, utils.GOB)
+				if err == nil { //Connected so no need to reiterate
+					break
+				}
+				time.Sleep(delay())
+			}
+			if err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<CDRS> Could not connect to stats server: %s", err.Error()))
 				exitChan <- true
 				return
 			}
-			time.Sleep(time.Duration(i) * time.Second)
+			statsConn = &engine.ProxyStats{Client: client}
 		}
 	}
-	engine.SetHistoryScribe(scribeServer) // scribeServer comes from global variable
-	return
+
+	cdrServer, _ = engine.NewCdrServer(cfg, cdrDb, raterConn, statsConn)
+	engine.Logger.Info("Registering CDRS HTTP Handlers.")
+	cdrServer.RegisterHanlersToServer(server)
+	engine.Logger.Info("Registering CDRS RPC service.")
+	cdrSrv := v1.CdrsV1{CdrSrv: cdrServer}
+	server.RpcRegister(&cdrSrv)
+	server.RpcRegister(&v2.CdrsV2{CdrsV1: cdrSrv})
+	// Make the cdr servers available for internal communication
+	responder.CdrSrv = cdrServer
+	close(doneChan)
 }
 
 // Starts the rpc server, waiting for the necessary components to finish their tasks
@@ -270,26 +354,6 @@ func serveHttp(httpWaitChans []chan struct{}) {
 	server.ServeHTTP(cfg.HTTPListen)
 }
 
-func checkConfigSanity() error {
-	if cfg.SMEnabled && cfg.RaterEnabled && cfg.RaterBalancer != "" {
-		engine.Logger.Crit("The session manager must not be enabled on a worker engine (change [engine]/balancer to disabled)!")
-		return errors.New("SessionManager on Worker")
-	}
-	if cfg.BalancerEnabled && cfg.RaterEnabled && cfg.RaterBalancer != "" {
-		engine.Logger.Crit("The balancer is enabled so it cannot connect to another balancer (change rater/balancer to disabled)!")
-		return errors.New("Improperly configured balancer")
-	}
-	if cfg.CDRSEnabled && cfg.CDRSMediator == utils.INTERNAL && !cfg.MediatorEnabled {
-		engine.Logger.Crit("CDRS cannot connect to mediator, Mediator not enabled in configuration!")
-		return errors.New("Internal Mediator required by CDRS")
-	}
-	if cfg.HistoryServerEnabled && cfg.HistoryServer == utils.INTERNAL && !cfg.HistoryServerEnabled {
-		engine.Logger.Crit("The history agent is enabled and internal and history server is disabled!")
-		return errors.New("Improperly configured history service")
-	}
-	return nil
-}
-
 func writePid() {
 	engine.Logger.Info(*pidFile)
 	f, err := os.Create(*pidFile)
@@ -311,9 +375,18 @@ func main() {
 	if *pidFile != "" {
 		writePid()
 	}
-	// runtime.GOMAXPROCS(runtime.NumCPU())   // For now it slows down computing due to CPU management, to be reviewed in future Go releases
-
-	cfg, err = config.NewCGRConfigFromFile(cfgPath)
+	if !*singlecpu {
+		runtime.GOMAXPROCS(runtime.NumCPU()) // For now it slows down computing due to CPU management, to be reviewed in future Go releases
+	}
+	if *cpuprofile != "" {
+		f, err := os.Create(*cpuprofile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+	cfg, err = config.NewCGRConfigFromFolder(*cfgDir)
 	if err != nil {
 		engine.Logger.Crit(fmt.Sprintf("Could not parse config: %s exiting!", err))
 		return
@@ -328,64 +401,49 @@ func main() {
 	if *cdrsEnabled {
 		cfg.CDRSEnabled = *cdrsEnabled
 	}
-	if *cdrcEnabled {
-		cfg.CdrcEnabled = *cdrcEnabled
-	}
-	if *mediatorEnabled {
-		cfg.MediatorEnabled = *mediatorEnabled
-	}
-
-	// some consitency checks
-	errCfg := checkConfigSanity()
-	if errCfg != nil {
-		engine.Logger.Crit(errCfg.Error())
-		return
-	}
 	var ratingDb engine.RatingStorage
 	var accountDb engine.AccountingStorage
 	var logDb engine.LogStorage
 	var loadDb engine.LoadStorage
 	var cdrDb engine.CdrStorage
-	ratingDb, err = engine.ConfigureRatingStorage(cfg.RatingDBType, cfg.RatingDBHost, cfg.RatingDBPort,
-		cfg.RatingDBName, cfg.RatingDBUser, cfg.RatingDBPass, cfg.DBDataEncoding)
-	if err != nil { // Cannot configure getter database, show stopper
-		engine.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err))
-		return
-	}
-	defer ratingDb.Close()
-	engine.SetRatingStorage(ratingDb)
-	accountDb, err = engine.ConfigureAccountingStorage(cfg.AccountDBType, cfg.AccountDBHost, cfg.AccountDBPort,
-		cfg.AccountDBName, cfg.AccountDBUser, cfg.AccountDBPass, cfg.DBDataEncoding)
-	if err != nil { // Cannot configure getter database, show stopper
-		engine.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err))
-		return
-	}
-	defer accountDb.Close()
-	engine.SetAccountingStorage(accountDb)
-
-	if cfg.StorDBType == SAME {
-		logDb = ratingDb.(engine.LogStorage)
-	} else {
-		logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort,
-			cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding)
-		if err != nil { // Cannot configure logger database, show stopper
-			engine.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting!", err))
+	if cfg.RaterEnabled || cfg.SchedulerEnabled { // Only connect to dataDb if required
+		ratingDb, err = engine.ConfigureRatingStorage(cfg.TpDbType, cfg.TpDbHost, cfg.TpDbPort,
+			cfg.TpDbName, cfg.TpDbUser, cfg.TpDbPass, cfg.DBDataEncoding)
+		if err != nil { // Cannot configure getter database, show stopper
+			engine.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err))
 			return
 		}
+		defer ratingDb.Close()
+		engine.SetRatingStorage(ratingDb)
+		accountDb, err = engine.ConfigureAccountingStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort,
+			cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding)
+		if err != nil { // Cannot configure getter database, show stopper
+			engine.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err))
+			return
+		}
+		defer accountDb.Close()
+		engine.SetAccountingStorage(accountDb)
+	}
+	if cfg.RaterEnabled || cfg.CDRSEnabled || cfg.SchedulerEnabled { // Only connect to storDb if necessary
+		if cfg.StorDBType == SAME {
+			logDb = ratingDb.(engine.LogStorage)
+		} else {
+			logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort,
+				cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns)
+			if err != nil { // Cannot configure logger database, show stopper
+				engine.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting!", err))
+				return
+			}
+		}
+		defer logDb.Close()
+		engine.SetStorageLogger(logDb)
+		// loadDb,cdrDb and logDb are all mapped on the same stordb storage
+		loadDb = logDb.(engine.LoadStorage)
+		cdrDb = logDb.(engine.CdrStorage)
+		engine.SetCdrStorage(cdrDb)
 	}
-	defer logDb.Close()
-	engine.SetStorageLogger(logDb)
-	// loadDb,cdrDb and logDb are all mapped on the same stordb storage
-	loadDb = logDb.(engine.LoadStorage)
-	cdrDb = logDb.(engine.CdrStorage)
 
 	engine.SetRoundingDecimals(cfg.RoundingDecimals)
-	if cfg.SMDebitInterval > 0 {
-		if dp, err := time.ParseDuration(fmt.Sprintf("%vs", cfg.SMDebitInterval)); err == nil {
-			engine.SetDebitPeriod(dp)
-		}
-	}
-
 	stopHandled := false
 
 	// Async starts here
@@ -405,23 +463,101 @@ func main() {
 		go stopRaterSignalHandler()
 		stopHandled = true
 	}
-
 	if cfg.CDRStatsEnabled { // Init it here so we make it availabe to the Apier
-		cdrStats = engine.NewStats(ratingDb)
-		if cfg.CDRStatConfig != nil && len(cfg.CDRStatConfig.Metrics) != 0 {
-			cdrStats.AddQueue(engine.NewCdrStatsFromCdrStatsCfg(cfg.CDRStatConfig), nil)
-		}
+		cdrStats = engine.NewStats(ratingDb, accountDb, cfg.CDRStatsSaveInterval)
 		server.RpcRegister(cdrStats)
-		server.RpcRegister(&apier.CDRStatsV1{cdrStats}) // Public APIs
+		server.RpcRegister(&v1.CDRStatsV1{CdrStats: cdrStats}) // Public APIs
 	}
 
-	responder := &engine.Responder{ExitChan: exitChan}
-	apierRpc := &apier.ApierV1{StorDb: loadDb, RatingDb: ratingDb, AccountDb: accountDb, CdrDb: cdrDb, LogDb: logDb, Config: cfg, Responder: responder, CdrStatsSrv: cdrStats}
+	if cfg.PubSubServerEnabled {
+		pubSubServer = engine.NewPubSub(accountDb, cfg.HttpSkipTlsVerify)
+		server.RpcRegisterName("PubSubV1", pubSubServer)
+	}
+
+	if cfg.HistoryServerEnabled {
+		scribeServer, err = history.NewFileScribe(cfg.HistoryDir, cfg.HistorySaveInterval)
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<HistoryServer> Could not start, error: %s", err.Error()))
+			exitChan <- true
+		}
+		server.RpcRegisterName("ScribeV1", scribeServer)
+	}
+	if cfg.UserServerEnabled {
+		userServer, err = engine.NewUserMap(accountDb, cfg.UserServerIndexes)
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<UsersService> Could not start, error: %s", err.Error()))
+			exitChan <- true
+		}
+		server.RpcRegisterName("UsersV1", userServer)
+	}
+
+	// Register session manager service // FixMe: make sure this is thread safe
+	if cfg.SmFsConfig.Enabled || cfg.SmKamConfig.Enabled || cfg.SmOsipsConfig.Enabled { // Register SessionManagerV1 service
+		smRpc = new(v1.SessionManagerV1)
+		server.RpcRegister(smRpc)
+	}
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if cfg.RaterCdrStats != "" && cfg.RaterCdrStats != utils.INTERNAL {
+			if cdrStats, err = engine.NewProxyStats(cfg.RaterCdrStats, cfg.ConnectAttempts, -1); err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<CdrStats> Could not connect to the server, error: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if cfg.RaterHistoryServer != "" && cfg.RaterHistoryServer != utils.INTERNAL {
+			if scribeServer, err = history.NewProxyScribe(cfg.RaterHistoryServer, cfg.ConnectAttempts, -1); err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<HistoryServer> Could not connect to the server, error: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+		}
+		engine.SetHistoryScribe(scribeServer)
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if cfg.RaterPubSubServer != "" && cfg.RaterPubSubServer != utils.INTERNAL {
+			if pubSubServer, err = engine.NewProxyPubSub(cfg.RaterPubSubServer, cfg.ConnectAttempts, -1); err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<PubSubServer> Could not connect to the server, error: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+		}
+		engine.SetPubSub(pubSubServer)
+	}()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if cfg.RaterUserServer != "" && cfg.RaterUserServer != utils.INTERNAL {
+			if userServer, err = engine.NewProxyUserService(cfg.RaterUserServer, cfg.ConnectAttempts, -1); err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<UserServer> Could not connect to the server, error: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+		}
+		engine.SetUserService(userServer)
+	}()
+	wg.Wait()
+
+	responder := &engine.Responder{ExitChan: exitChan, Stats: cdrStats}
+	apierRpcV1 := &v1.ApierV1{StorDb: loadDb, RatingDb: ratingDb, AccountDb: accountDb, CdrDb: cdrDb, LogDb: logDb, Config: cfg, Responder: responder, CdrStatsSrv: cdrStats, Users: userServer}
+	apierRpcV2 := &v2.ApierV2{
+		ApierV1: v1.ApierV1{StorDb: loadDb, RatingDb: ratingDb, AccountDb: accountDb, CdrDb: cdrDb, LogDb: logDb, Config: cfg, Responder: responder, CdrStatsSrv: cdrStats, Users: userServer}}
 
 	if cfg.RaterEnabled && !cfg.BalancerEnabled && cfg.RaterBalancer != utils.INTERNAL {
 		engine.Logger.Info("Registering Rater service")
 		server.RpcRegister(responder)
-		server.RpcRegister(apierRpc)
+		server.RpcRegister(apierRpcV1)
+		server.RpcRegister(apierRpcV2)
 	}
 
 	if cfg.BalancerEnabled {
@@ -430,7 +566,8 @@ func main() {
 		stopHandled = true
 		responder.Bal = bal
 		server.RpcRegister(responder)
-		server.RpcRegister(apierRpc)
+		server.RpcRegister(apierRpcV1)
+		server.RpcRegister(apierRpcV2)
 		if cfg.RaterEnabled {
 			engine.Logger.Info("<Balancer> Registering internal rater")
 			bal.AddClient("local", new(engine.ResponderWorker))
@@ -445,60 +582,48 @@ func main() {
 		engine.Logger.Info("Starting CGRateS Scheduler.")
 		go func() {
 			sched := scheduler.NewScheduler()
-			go reloadSchedulerSingnalHandler(sched, accountDb)
-			apierRpc.Sched = sched
-			sched.LoadActionTimings(accountDb)
+			go reloadSchedulerSingnalHandler(sched, ratingDb)
+			apierRpcV1.Sched = sched
+			apierRpcV2.Sched = sched
+			sched.LoadActionPlans(ratingDb)
 			sched.Loop()
 		}()
 	}
 
-	var histServChan chan struct{} // Will be initialized only if the server starts
-	if cfg.HistoryServerEnabled {
-		histServChan = make(chan struct{})
-		rpcWait = append(rpcWait, histServChan)
-		go startHistoryServer(histServChan)
-	}
-
-	if cfg.HistoryAgentEnabled {
-		engine.Logger.Info("Starting CGRateS History Agent.")
-		go startHistoryAgent(histServChan)
-	}
-
-	var medChan chan struct{}
-	if cfg.MediatorEnabled {
-		engine.Logger.Info("Starting CGRateS Mediator service.")
-		medChan = make(chan struct{})
-		go startMediator(responder, logDb, cdrDb, cacheChan, medChan)
-	}
-
 	var cdrsChan chan struct{}
 	if cfg.CDRSEnabled {
 		engine.Logger.Info("Starting CGRateS CDRS service.")
 		cdrsChan = make(chan struct{})
 		httpWait = append(httpWait, cdrsChan)
-		go startCDRS(responder, cdrDb, medChan, cdrsChan)
+		go startCDRS(logDb, cdrDb, responder, cacheChan, cdrsChan)
 	}
 
-	if cfg.SMEnabled {
-		engine.Logger.Info("Starting CGRateS SessionManager service.")
-		go startSessionManager(responder, logDb, cacheChan)
+	if cfg.SmFsConfig.Enabled {
+		engine.Logger.Info("Starting CGRateS SM-FreeSWITCH service.")
+		go startSmFreeSWITCH(responder, cdrDb, cacheChan)
 		// close all sessions on shutdown
 		go shutdownSessionmanagerSingnalHandler()
 	}
-	var cdrcEnabled bool
-	if cfg.CdrcEnabled { // Start default cdrc configured in csv here
-		cdrcEnabled = true
-		go startCdrc(cdrsChan, cfg.CdrcCdrs, cfg.CdrcCdrType, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.CdrcSourceId, cfg.CdrcRunDelay, cfg.CdrcCsvSep, cfg.CdrcCdrFields)
+	if cfg.SmKamConfig.Enabled {
+		engine.Logger.Info("Starting CGRateS SM-Kamailio service.")
+		go startSmKamailio(responder, cdrDb, cacheChan)
 	}
-	if cfg.XmlCfgDocument != nil {
-		for _, xmlCdrc := range cfg.XmlCfgDocument.GetCdrcCfgs("") {
-			if !xmlCdrc.Enabled {
-				continue
-			}
-			cdrcEnabled = true
-			go startCdrc(cdrsChan, xmlCdrc.CdrsAddress, xmlCdrc.CdrType, xmlCdrc.CdrInDir, xmlCdrc.CdrOutDir,
-				xmlCdrc.CdrSourceId, time.Duration(xmlCdrc.RunDelay), xmlCdrc.CsvSeparator, xmlCdrc.CdrRSRFields())
+	if cfg.SmOsipsConfig.Enabled {
+		engine.Logger.Info("Starting CGRateS SM-OpenSIPS service.")
+		go startSmOpenSIPS(responder, cdrDb, cacheChan)
+	}
+	var cdrcEnabled bool
+	for _, cdrcCfgs := range cfg.CdrcProfiles {
+		var cdrcCfg *config.CdrcConfig
+		for _, cdrcCfg = range cdrcCfgs { // Take a random config out since they should be the same
+			break
 		}
+		if cdrcCfg.Enabled == false {
+			continue // Ignore not enabled
+		} else if !cdrcEnabled {
+			cdrcEnabled = true // Mark that at least one cdrc service is active
+		}
+		go startCdrc(responder, cdrsChan, cdrcCfgs, cfg.HttpSkipTlsVerify, cfg.ConfigReloads[utils.CDRC])
 	}
 	if cdrcEnabled {
 		engine.Logger.Info("Starting CGRateS CDR client.")
@@ -507,7 +632,6 @@ func main() {
 	// Start the servers
 	go serveRpc(rpcWait)
 	go serveHttp(httpWait)
-
 	<-exitChan
 
 	if *pidFile != "" {

cmd/cgr-engine/engine_test.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

cmd/cgr-engine/registration.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -48,6 +48,10 @@ func generalSignalHandler() {
 
 	sig := <-c
 	engine.Logger.Info(fmt.Sprintf("Caught signal %v, shuting down cgr-engine\n", sig))
+	var dummyInt int
+	if cdrStats != nil {
+		cdrStats.Stop(dummyInt, &dummyInt)
+	}
 	exitChan <- true
 }
 
@@ -61,6 +65,8 @@ func stopRaterSignalHandler() {
 
 	engine.Logger.Info(fmt.Sprintf("Caught signal %v, unregistering from balancer\n", sig))
 	unregisterFromBalancer()
+	var dummyInt int
+	cdrStats.Stop(dummyInt, &dummyInt)
 	exitChan <- true
 }
 
@@ -104,14 +110,14 @@ func registerToBalancer() {
 }
 
 // Listens for the HUP system signal and gracefuly reloads the timers from database.
-func reloadSchedulerSingnalHandler(sched *scheduler.Scheduler, getter engine.AccountingStorage) {
+func reloadSchedulerSingnalHandler(sched *scheduler.Scheduler, getter engine.RatingStorage) {
 	for {
 		c := make(chan os.Signal)
 		signal.Notify(c, syscall.SIGHUP)
 		sig := <-c
 
 		engine.Logger.Info(fmt.Sprintf("Caught signal %v, reloading action timings.\n", sig))
-		sched.LoadActionTimings(getter)
+		sched.LoadActionPlans(getter)
 		// check the tip of the queue for new actions
 		sched.Restart()
 	}
@@ -125,7 +131,7 @@ func shutdownSessionmanagerSingnalHandler() {
 	signal.Notify(c, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT)
 	<-c
 
-	if sm != nil {
+	for _, sm := range sms {
 		if err := sm.Shutdown(); err != nil {
 			engine.Logger.Warning(fmt.Sprintf("<SessionManager> %s", err))
 		}

cmd/cgr-loader/cgr-loader.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -34,19 +34,19 @@ import (
 var (
 	//separator = flag.String("separator", ",", "Default field separator")
 	cgrConfig, _  = config.NewDefaultCGRConfig()
-	ratingdb_type = flag.String("ratingdb_type", cgrConfig.RatingDBType, "The type of the RatingDb database <redis>")
-	ratingdb_host = flag.String("ratingdb_host", cgrConfig.RatingDBHost, "The RatingDb host to connect to.")
-	ratingdb_port = flag.String("ratingdb_port", cgrConfig.RatingDBPort, "The RatingDb port to bind to.")
-	ratingdb_name = flag.String("ratingdb_name", cgrConfig.RatingDBName, "The name/number of the RatingDb to connect to.")
-	ratingdb_user = flag.String("ratingdb_user", cgrConfig.RatingDBUser, "The RatingDb user to sign in as.")
-	ratingdb_pass = flag.String("ratingdb_passwd", cgrConfig.RatingDBPass, "The RatingDb user's password.")
+	ratingdb_type = flag.String("ratingdb_type", cgrConfig.TpDbType, "The type of the RatingDb database <redis>")
+	ratingdb_host = flag.String("ratingdb_host", cgrConfig.TpDbHost, "The RatingDb host to connect to.")
+	ratingdb_port = flag.String("ratingdb_port", cgrConfig.TpDbPort, "The RatingDb port to bind to.")
+	ratingdb_name = flag.String("ratingdb_name", cgrConfig.TpDbName, "The name/number of the RatingDb to connect to.")
+	ratingdb_user = flag.String("ratingdb_user", cgrConfig.TpDbUser, "The RatingDb user to sign in as.")
+	ratingdb_pass = flag.String("ratingdb_passwd", cgrConfig.TpDbPass, "The RatingDb user's password.")
 
-	accountdb_type = flag.String("accountdb_type", cgrConfig.AccountDBType, "The type of the AccountingDb database <redis>")
-	accountdb_host = flag.String("accountdb_host", cgrConfig.AccountDBHost, "The AccountingDb host to connect to.")
-	accountdb_port = flag.String("accountdb_port", cgrConfig.AccountDBPort, "The AccountingDb port to bind to.")
-	accountdb_name = flag.String("accountdb_name", cgrConfig.AccountDBName, "The name/number of the AccountingDb to connect to.")
-	accountdb_user = flag.String("accountdb_user", cgrConfig.AccountDBUser, "The AccountingDb user to sign in as.")
-	accountdb_pass = flag.String("accountdb_passwd", cgrConfig.AccountDBPass, "The AccountingDb user's password.")
+	accountdb_type = flag.String("accountdb_type", cgrConfig.DataDbType, "The type of the AccountingDb database <redis>")
+	accountdb_host = flag.String("accountdb_host", cgrConfig.DataDbHost, "The AccountingDb host to connect to.")
+	accountdb_port = flag.String("accountdb_port", cgrConfig.DataDbPort, "The AccountingDb port to bind to.")
+	accountdb_name = flag.String("accountdb_name", cgrConfig.DataDbName, "The name/number of the AccountingDb to connect to.")
+	accountdb_user = flag.String("accountdb_user", cgrConfig.DataDbUser, "The AccountingDb user to sign in as.")
+	accountdb_pass = flag.String("accountdb_passwd", cgrConfig.DataDbPass, "The AccountingDb user's password.")
 
 	stor_db_type = flag.String("stordb_type", cgrConfig.StorDBType, "The type of the storDb database <mysql>")
 	stor_db_host = flag.String("stordb_host", cgrConfig.StorDBHost, "The storDb host to connect to.")
@@ -63,12 +63,14 @@ var (
 	version         = flag.Bool("version", false, "Prints the application version.")
 	verbose         = flag.Bool("verbose", false, "Enable detailed verbose logging output")
 	dryRun          = flag.Bool("dry_run", false, "When true will not save loaded data to dataDb but just parse it for consistency and errors.")
+	validate        = flag.Bool("validate", false, "When true will run various check on the loaded data to check for structural errors")
 	stats           = flag.Bool("stats", false, "Generates statsistics about given data.")
 	fromStorDb      = flag.Bool("from_stordb", false, "Load the tariff plan from storDb to dataDb")
 	toStorDb        = flag.Bool("to_stordb", false, "Import the tariff plan from files to storDb")
 	historyServer   = flag.String("history_server", cgrConfig.RPCGOBListen, "The history server address:port, empty to disable automaticautomatic  history archiving")
 	raterAddress    = flag.String("rater_address", cgrConfig.RPCGOBListen, "Rater service to contact for cache reloads, empty to disable automatic cache reloads")
 	cdrstatsAddress = flag.String("cdrstats_address", cgrConfig.RPCGOBListen, "CDRStats service to contact for data reloads, empty to disable automatic data reloads")
+	usersAddress    = flag.String("users_address", cgrConfig.RPCGOBListen, "Users service to contact for data reloads, empty to disable automatic data reloads")
 	runId           = flag.String("runid", "", "Uniquely identify an import/load, postpended to some automatic fields")
 )
 
@@ -82,17 +84,19 @@ func main() {
 	var ratingDb engine.RatingStorage
 	var accountDb engine.AccountingStorage
 	var storDb engine.LoadStorage
-	var rater, cdrstats *rpc.Client
-	var loader engine.TPLoader
+	var rater, cdrstats, users *rpc.Client
+	var loader engine.LoadReader
 	// Init necessary db connections, only if not already
 	if !*dryRun { // make sure we do not need db connections on dry run, also not importing into any stordb
 		if *fromStorDb {
 			ratingDb, errRatingDb = engine.ConfigureRatingStorage(*ratingdb_type, *ratingdb_host, *ratingdb_port, *ratingdb_name,
 				*ratingdb_user, *ratingdb_pass, *dbdata_encoding)
 			accountDb, errAccDb = engine.ConfigureAccountingStorage(*accountdb_type, *accountdb_host, *accountdb_port, *accountdb_name, *accountdb_user, *accountdb_pass, *dbdata_encoding)
-			storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding)
+			storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding,
+				cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns)
 		} else if *toStorDb { // Import from csv files to storDb
-			storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding)
+			storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding,
+				cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns)
 		} else { // Default load from csv files to dataDb
 			ratingDb, errRatingDb = engine.ConfigureRatingStorage(*ratingdb_type, *ratingdb_host, *ratingdb_port, *ratingdb_name,
 				*ratingdb_user, *ratingdb_pass, *dbdata_encoding)
@@ -114,7 +118,14 @@ func main() {
 			if *tpid == "" {
 				log.Fatal("TPid required, please define it via *-tpid* command argument.")
 			}
-			csvImporter := engine.TPCSVImporter{*tpid, storDb, *dataPath, ',', *verbose, *runId}
+			csvImporter := engine.TPCSVImporter{
+				TPid:     *tpid,
+				StorDb:   storDb,
+				DirPath:  *dataPath,
+				Sep:      ',',
+				Verbose:  *verbose,
+				ImportId: *runId,
+			}
 			if errImport := csvImporter.Run(); errImport != nil {
 				log.Fatal(errImport)
 			}
@@ -122,15 +133,15 @@ func main() {
 		}
 	}
 	if *fromStorDb { // Load Tariff Plan from storDb into dataDb
-		loader = engine.NewDbReader(storDb, ratingDb, accountDb, *tpid)
+		loader = storDb
 	} else { // Default load from csv files to dataDb
-		for fn, v := range engine.FileValidators {
+		/*for fn, v := range engine.FileValidators {
 			err := engine.ValidateCSVData(path.Join(*dataPath, fn), v.Rule)
 			if err != nil {
 				log.Fatal(err, "\n\t", v.Message)
 			}
-		}
-		loader = engine.NewFileCSVReader(ratingDb, accountDb, ',',
+		}*/
+		loader = engine.NewFileCSVStorage(',',
 			path.Join(*dataPath, utils.DESTINATIONS_CSV),
 			path.Join(*dataPath, utils.TIMINGS_CSV),
 			path.Join(*dataPath, utils.RATES_CSV),
@@ -144,25 +155,32 @@ func main() {
 			path.Join(*dataPath, utils.ACTION_TRIGGERS_CSV),
 			path.Join(*dataPath, utils.ACCOUNT_ACTIONS_CSV),
 			path.Join(*dataPath, utils.DERIVED_CHARGERS_CSV),
-			path.Join(*dataPath, utils.CDR_STATS_CSV))
+			path.Join(*dataPath, utils.CDR_STATS_CSV),
+			path.Join(*dataPath, utils.USERS_CSV))
 	}
-	err = loader.LoadAll()
+	tpReader := engine.NewTpReader(ratingDb, accountDb, loader, *tpid)
+	err = tpReader.LoadAll()
 	if err != nil {
 		log.Fatal(err)
 	}
 	if *stats {
-		loader.ShowStatistics()
+		tpReader.ShowStatistics()
+	}
+	if *validate {
+		if !tpReader.IsValid() {
+			return
+		}
 	}
 	if *dryRun { // We were just asked to parse the data, not saving it
 		return
 	}
 	if *historyServer != "" { // Init scribeAgent so we can store the differences
-		if scribeAgent, err := history.NewProxyScribe(*historyServer); err != nil {
+		if scribeAgent, err := history.NewProxyScribe(*historyServer, 3, 3); err != nil {
 			log.Fatalf("Could not connect to history server, error: %s. Make sure you have properly configured it via -history_server flag.", err.Error())
 			return
 		} else {
 			engine.SetHistoryScribe(scribeAgent)
-			defer scribeAgent.Client.Close()
+			//defer scribeAgent.Client.Close()
 		}
 	} else {
 		log.Print("WARNING: Rates history archiving is disabled!")
@@ -189,9 +207,22 @@ func main() {
 	} else {
 		log.Print("WARNING: CDRStats automatic data reload is disabled!")
 	}
+	if *usersAddress != "" { // Init connection to rater so we can reload it's data
+		if *usersAddress == *raterAddress {
+			users = rater
+		} else {
+			users, err = rpc.Dial("tcp", *usersAddress)
+			if err != nil {
+				log.Fatalf("Could not connect to Users API: %s", err.Error())
+				return
+			}
+		}
+	} else {
+		log.Print("WARNING: Users automatic data reload is disabled!")
+	}
 
 	// write maps to database
-	if err := loader.WriteToDatabase(*flush, *verbose); err != nil {
+	if err := tpReader.WriteToDatabase(*flush, *verbose); err != nil {
 		log.Fatal("Could not write to database: ", err)
 	}
 	if len(*historyServer) != 0 && *verbose {
@@ -200,15 +231,15 @@ func main() {
 	// Reload scheduler and cache
 	if rater != nil {
 		reply := ""
-		dstIds, _ := loader.GetLoadedIds(engine.DESTINATION_PREFIX)
-		rplIds, _ := loader.GetLoadedIds(engine.RATING_PLAN_PREFIX)
-		rpfIds, _ := loader.GetLoadedIds(engine.RATING_PROFILE_PREFIX)
-		actIds, _ := loader.GetLoadedIds(engine.ACTION_PREFIX)
-		shgIds, _ := loader.GetLoadedIds(engine.SHARED_GROUP_PREFIX)
-		rpAliases, _ := loader.GetLoadedIds(engine.RP_ALIAS_PREFIX)
-		accAliases, _ := loader.GetLoadedIds(engine.ACC_ALIAS_PREFIX)
-		lcrIds, _ := loader.GetLoadedIds(engine.LCR_PREFIX)
-		dcs, _ := loader.GetLoadedIds(engine.DERIVEDCHARGERS_PREFIX)
+		dstIds, _ := tpReader.GetLoadedIds(utils.DESTINATION_PREFIX)
+		rplIds, _ := tpReader.GetLoadedIds(utils.RATING_PLAN_PREFIX)
+		rpfIds, _ := tpReader.GetLoadedIds(utils.RATING_PROFILE_PREFIX)
+		actIds, _ := tpReader.GetLoadedIds(utils.ACTION_PREFIX)
+		shgIds, _ := tpReader.GetLoadedIds(utils.SHARED_GROUP_PREFIX)
+		rpAliases, _ := tpReader.GetLoadedIds(utils.RP_ALIAS_PREFIX)
+		accAliases, _ := tpReader.GetLoadedIds(utils.ACC_ALIAS_PREFIX)
+		lcrIds, _ := tpReader.GetLoadedIds(utils.LCR_PREFIX)
+		dcs, _ := tpReader.GetLoadedIds(utils.DERIVEDCHARGERS_PREFIX)
 		// Reload cache first since actions could be calling info from within
 		if *verbose {
 			log.Print("Reloading cache")
@@ -216,10 +247,20 @@ func main() {
 		if *flush {
 			dstIds, rplIds, rpfIds, rpAliases, lcrIds = nil, nil, nil, nil, nil // Should reload all these on flush
 		}
-		if err = rater.Call("ApierV1.ReloadCache", utils.ApiReloadCache{dstIds, rplIds, rpfIds, actIds, shgIds, rpAliases, accAliases, lcrIds, dcs}, &reply); err != nil {
+		if err = rater.Call("ApierV1.ReloadCache", utils.ApiReloadCache{
+			DestinationIds:   dstIds,
+			RatingPlanIds:    rplIds,
+			RatingProfileIds: rpfIds,
+			ActionIds:        actIds,
+			SharedGroupIds:   shgIds,
+			RpAliases:        rpAliases,
+			AccAliases:       accAliases,
+			LCRIds:           lcrIds,
+			DerivedChargers:  dcs,
+		}, &reply); err != nil {
 			log.Printf("WARNING: Got error on cache reload: %s\n", err.Error())
 		}
-		actTmgIds, _ := loader.GetLoadedIds(engine.ACTION_TIMING_PREFIX)
+		actTmgIds, _ := tpReader.GetLoadedIds(utils.ACTION_TIMING_PREFIX)
 		if len(actTmgIds) != 0 {
 			if *verbose {
 				log.Print("Reloading scheduler")
@@ -231,7 +272,7 @@ func main() {
 
 	}
 	if cdrstats != nil {
-		statsQueueIds, _ := loader.GetLoadedIds(engine.CDR_STATS_PREFIX)
+		statsQueueIds, _ := tpReader.GetLoadedIds(utils.CDR_STATS_PREFIX)
 		if *flush {
 			statsQueueIds = []string{} // Force reload all
 		}
@@ -245,4 +286,18 @@ func main() {
 			}
 		}
 	}
+
+	if users != nil {
+		userIds, _ := tpReader.GetLoadedIds(utils.USERS_PREFIX)
+		if len(userIds) > 0 {
+			if *verbose {
+				log.Print("Reloading Users data")
+			}
+			var reply string
+			if err := cdrstats.Call("UsersV1.ReloadUsers", "", &reply); err != nil {
+				log.Printf("WARNING: Failed reloading users data, error: %s\n", err.Error())
+			}
+
+		}
+	}
 }

cmd/cgr-tester/cgr-tester.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"log"
 	"net/rpc"
+	"net/rpc/jsonrpc"
 	"os"
 	"runtime"
 	"runtime/pprof"
@@ -39,18 +40,18 @@ var (
 	memprofile      = flag.String("memprofile", "", "write memory profile to this file")
 	runs            = flag.Int("runs", 10000, "stress cycle number")
 	parallel        = flag.Int("parallel", 0, "run n requests in parallel")
-	ratingdb_type   = flag.String("ratingdb_type", cgrConfig.RatingDBType, "The type of the RatingDb database <redis>")
-	ratingdb_host   = flag.String("ratingdb_host", cgrConfig.RatingDBHost, "The RatingDb host to connect to.")
-	ratingdb_port   = flag.String("ratingdb_port", cgrConfig.RatingDBPort, "The RatingDb port to bind to.")
-	ratingdb_name   = flag.String("ratingdb_name", cgrConfig.RatingDBName, "The name/number of the RatingDb to connect to.")
-	ratingdb_user   = flag.String("ratingdb_user", cgrConfig.RatingDBUser, "The RatingDb user to sign in as.")
-	ratingdb_pass   = flag.String("ratingdb_passwd", cgrConfig.RatingDBPass, "The RatingDb user's password.")
-	accountdb_type  = flag.String("accountdb_type", cgrConfig.AccountDBType, "The type of the AccountingDb database <redis>")
-	accountdb_host  = flag.String("accountdb_host", cgrConfig.AccountDBHost, "The AccountingDb host to connect to.")
-	accountdb_port  = flag.String("accountdb_port", cgrConfig.AccountDBPort, "The AccountingDb port to bind to.")
-	accountdb_name  = flag.String("accountdb_name", cgrConfig.AccountDBName, "The name/number of the AccountingDb to connect to.")
-	accountdb_user  = flag.String("accountdb_user", cgrConfig.AccountDBUser, "The AccountingDb user to sign in as.")
-	accountdb_pass  = flag.String("accountdb_passwd", cgrConfig.AccountDBPass, "The AccountingDb user's password.")
+	ratingdb_type   = flag.String("ratingdb_type", cgrConfig.TpDbType, "The type of the RatingDb database <redis>")
+	ratingdb_host   = flag.String("ratingdb_host", cgrConfig.TpDbHost, "The RatingDb host to connect to.")
+	ratingdb_port   = flag.String("ratingdb_port", cgrConfig.TpDbPort, "The RatingDb port to bind to.")
+	ratingdb_name   = flag.String("ratingdb_name", cgrConfig.TpDbName, "The name/number of the RatingDb to connect to.")
+	ratingdb_user   = flag.String("ratingdb_user", cgrConfig.TpDbUser, "The RatingDb user to sign in as.")
+	ratingdb_pass   = flag.String("ratingdb_passwd", cgrConfig.TpDbPass, "The RatingDb user's password.")
+	accountdb_type  = flag.String("accountdb_type", cgrConfig.DataDbType, "The type of the AccountingDb database <redis>")
+	accountdb_host  = flag.String("accountdb_host", cgrConfig.DataDbHost, "The AccountingDb host to connect to.")
+	accountdb_port  = flag.String("accountdb_port", cgrConfig.DataDbPort, "The AccountingDb port to bind to.")
+	accountdb_name  = flag.String("accountdb_name", cgrConfig.DataDbName, "The name/number of the AccountingDb to connect to.")
+	accountdb_user  = flag.String("accountdb_user", cgrConfig.DataDbUser, "The AccountingDb user to sign in as.")
+	accountdb_pass  = flag.String("accountdb_passwd", cgrConfig.DataDbPass, "The AccountingDb user's password.")
 	dbdata_encoding = flag.String("dbdata_encoding", cgrConfig.DBDataEncoding, "The encoding used to store object data in strings.")
 	raterAddress    = flag.String("rater_address", "", "Rater address for remote tests. Empty for internal rater.")
 	tor             = flag.String("tor", utils.VOICE, "The type of record to use in queries.")
@@ -58,6 +59,7 @@ var (
 	tenant          = flag.String("tenant", "cgrates.org", "The type of record to use in queries.")
 	subject         = flag.String("subject", "1001", "The rating subject to use in queries.")
 	destination     = flag.String("destination", "1002", "The destination to use in queries.")
+	json            = flag.Bool("json", false, "Use JSON RPC")
 
 	nilDuration = time.Duration(0)
 )
@@ -75,7 +77,7 @@ func durInternalRater(cd *engine.CallDescriptor) (time.Duration, error) {
 	}
 	defer accountDb.Close()
 	engine.SetAccountingStorage(accountDb)
-	if err := ratingDb.CacheRating(nil, nil, nil, nil, nil); err != nil {
+	if err := ratingDb.CacheAll(); err != nil {
 		return nilDuration, fmt.Errorf("Cache rating error: %s", err.Error())
 	}
 	log.Printf("Runnning %d cycles...", *runs)
@@ -107,9 +109,16 @@ func durInternalRater(cd *engine.CallDescriptor) (time.Duration, error) {
 
 func durRemoteRater(cd *engine.CallDescriptor) (time.Duration, error) {
 	result := engine.CallCost{}
-	client, err := rpc.Dial("tcp", *raterAddress)
+	var client *rpc.Client
+	var err error
+	if *json {
+		client, err = jsonrpc.Dial("tcp", *raterAddress)
+	} else {
+		client, err = rpc.Dial("tcp", *raterAddress)
+	}
+
 	if err != nil {
-		return nilDuration, fmt.Errorf("Could not connect to engine: ", err.Error())
+		return nilDuration, fmt.Errorf("Could not connect to engine: %s", err.Error())
 	}
 	defer client.Close()
 	start := time.Now()

config/cdrcconfig.go

@@ -0,0 +1,146 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"time"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+type CdrcConfig struct {
+	Enabled                 bool            // Enable/Disable the profile
+	DryRun                  bool            // Do not post CDRs to the server
+	Cdrs                    string          // The address where CDRs can be reached
+	CdrFormat               string          // The type of CDR file to process <csv|opensips_flatstore>
+	FieldSeparator          rune            // The separator to use when reading csvs
+	DataUsageMultiplyFactor float64         // Conversion factor for data usage
+	RunDelay                time.Duration   // Delay between runs, 0 for inotify driven requests
+	MaxOpenFiles            int             // Maximum number of files opened simultaneously
+	CdrInDir                string          // Folder to process CDRs from
+	CdrOutDir               string          // Folder to move processed CDRs to
+	FailedCallsPrefix       string          // Used in case of flatstore CDRs to avoid searching for BYE records
+	CdrSourceId             string          // Source identifier for the processed CDRs
+	CdrFilter               utils.RSRFields // Filter CDR records to import
+	PartialRecordCache      time.Duration   // Duration to cache partial records when not pairing
+	HeaderFields            []*CfgCdrField
+	ContentFields           []*CfgCdrField
+	TrailerFields           []*CfgCdrField
+}
+
+func (self *CdrcConfig) loadFromJsonCfg(jsnCfg *CdrcJsonCfg) error {
+	if jsnCfg == nil {
+		return nil
+	}
+	var err error
+	if jsnCfg.Enabled != nil {
+		self.Enabled = *jsnCfg.Enabled
+	}
+	if jsnCfg.Dry_run != nil {
+		self.DryRun = *jsnCfg.Dry_run
+	}
+	if jsnCfg.Cdrs != nil {
+		self.Cdrs = *jsnCfg.Cdrs
+	}
+	if jsnCfg.Cdr_format != nil {
+		self.CdrFormat = *jsnCfg.Cdr_format
+	}
+	if jsnCfg.Field_separator != nil && len(*jsnCfg.Field_separator) > 0 {
+		sepStr := *jsnCfg.Field_separator
+		self.FieldSeparator = rune(sepStr[0])
+	}
+	if jsnCfg.Data_usage_multiply_factor != nil {
+		self.DataUsageMultiplyFactor = *jsnCfg.Data_usage_multiply_factor
+	}
+	if jsnCfg.Run_delay != nil {
+		self.RunDelay = time.Duration(*jsnCfg.Run_delay) * time.Second
+	}
+	if jsnCfg.Max_open_files != nil {
+		self.MaxOpenFiles = *jsnCfg.Max_open_files
+	}
+	if jsnCfg.Cdr_in_dir != nil {
+		self.CdrInDir = *jsnCfg.Cdr_in_dir
+	}
+	if jsnCfg.Cdr_out_dir != nil {
+		self.CdrOutDir = *jsnCfg.Cdr_out_dir
+	}
+	if jsnCfg.Failed_calls_prefix != nil {
+		self.FailedCallsPrefix = *jsnCfg.Failed_calls_prefix
+	}
+	if jsnCfg.Cdr_source_id != nil {
+		self.CdrSourceId = *jsnCfg.Cdr_source_id
+	}
+	if jsnCfg.Cdr_filter != nil {
+		if self.CdrFilter, err = utils.ParseRSRFields(*jsnCfg.Cdr_filter, utils.INFIELD_SEP); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Partial_record_cache != nil {
+		if self.PartialRecordCache, err = utils.ParseDurationWithSecs(*jsnCfg.Partial_record_cache); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Header_fields != nil {
+		if self.HeaderFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Header_fields); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Content_fields != nil {
+		if self.ContentFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Content_fields); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Trailer_fields != nil {
+		if self.TrailerFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Trailer_fields); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Clone itself into a new CdrcConfig
+func (self *CdrcConfig) Clone() *CdrcConfig {
+	clnCdrc := new(CdrcConfig)
+	clnCdrc.Enabled = self.Enabled
+	clnCdrc.Cdrs = self.Cdrs
+	clnCdrc.CdrFormat = self.CdrFormat
+	clnCdrc.FieldSeparator = self.FieldSeparator
+	clnCdrc.DataUsageMultiplyFactor = self.DataUsageMultiplyFactor
+	clnCdrc.RunDelay = self.RunDelay
+	clnCdrc.MaxOpenFiles = self.MaxOpenFiles
+	clnCdrc.CdrInDir = self.CdrInDir
+	clnCdrc.CdrOutDir = self.CdrOutDir
+	clnCdrc.CdrSourceId = self.CdrSourceId
+	clnCdrc.HeaderFields = make([]*CfgCdrField, len(self.HeaderFields))
+	clnCdrc.ContentFields = make([]*CfgCdrField, len(self.ContentFields))
+	clnCdrc.TrailerFields = make([]*CfgCdrField, len(self.TrailerFields))
+	for idx, fld := range self.HeaderFields {
+		clonedVal := *fld
+		clnCdrc.HeaderFields[idx] = &clonedVal
+	}
+	for idx, fld := range self.ContentFields {
+		clonedVal := *fld
+		clnCdrc.ContentFields[idx] = &clonedVal
+	}
+	for idx, fld := range self.TrailerFields {
+		clonedVal := *fld
+		clnCdrc.TrailerFields[idx] = &clonedVal
+	}
+	return clnCdrc
+}

config/cdrcconfig_test.go

@@ -0,0 +1,19 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config

config/cdreconfig.go

@@ -1,6 +1,6 @@
 /*
 Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
+Copyright (C) ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -18,227 +18,109 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 
 package config
 
-import (
-	"errors"
-	"github.com/cgrates/cgrates/utils"
-)
-
-// Converts a list of field identifiers into proper CDR field content
-func NewCdreCdrFieldsFromIds(withFixedWith bool, fldsIds ...string) ([]*CdreCdrField, error) {
-	cdrFields := make([]*CdreCdrField, len(fldsIds))
-	for idx, fldId := range fldsIds {
-		if parsedRsr, err := utils.NewRSRField(fldId); err != nil {
-			return nil, err
-		} else {
-			cdrFld := &CdreCdrField{Name: fldId, Type: utils.CDRFIELD, Value: fldId, valueAsRsrField: parsedRsr}
-			if err := cdrFld.setDefaultFieldProperties(withFixedWith); err != nil { // Set default fixed width properties to be used later if needed
-				return nil, err
-
-			}
-			cdrFields[idx] = cdrFld
-		}
-	}
-	return cdrFields, nil
-}
-
-func NewDefaultCdreConfig() (*CdreConfig, error) {
-	cdreCfg := new(CdreConfig)
-	if err := cdreCfg.setDefaults(); err != nil {
-		return nil, err
-	}
-	return cdreCfg, nil
-}
-
 // One instance of CdrExporter
 type CdreConfig struct {
-	CdrFormat               string
-	FieldSeparator          rune
-	DataUsageMultiplyFactor float64
-	CostMultiplyFactor      float64
-	CostRoundingDecimals    int
-	CostShiftDigits         int
-	MaskDestId              string
-	MaskLength              int
-	ExportDir               string
-	HeaderFields            []*CdreCdrField
-	ContentFields           []*CdreCdrField
-	TrailerFields           []*CdreCdrField
+	CdrFormat                  string
+	FieldSeparator             rune
+	DataUsageMultiplyFactor    float64
+	SmsUsageMultiplyFactor     float64
+	GenericUsageMultiplyFactor float64
+	CostMultiplyFactor         float64
+	CostRoundingDecimals       int
+	CostShiftDigits            int
+	MaskDestId                 string
+	MaskLength                 int
+	ExportDir                  string
+	HeaderFields               []*CfgCdrField
+	ContentFields              []*CfgCdrField
+	TrailerFields              []*CfgCdrField
 }
 
-// Set here defaults
-func (cdreCfg *CdreConfig) setDefaults() error {
-	cdreCfg.CdrFormat = utils.CSV
-	cdreCfg.FieldSeparator = utils.CSV_SEP
-	cdreCfg.DataUsageMultiplyFactor = 0.0
-	cdreCfg.CostMultiplyFactor = 0.0
-	cdreCfg.CostRoundingDecimals = -1
-	cdreCfg.CostShiftDigits = 0
-	cdreCfg.MaskDestId = ""
-	cdreCfg.MaskLength = 0
-	cdreCfg.ExportDir = "/var/log/cgrates/cdre"
-	if flds, err := NewCdreCdrFieldsFromIds(false, utils.CGRID, utils.MEDI_RUNID, utils.TOR, utils.ACCID, utils.REQTYPE, utils.DIRECTION, utils.TENANT,
-		utils.CATEGORY, utils.ACCOUNT, utils.SUBJECT, utils.DESTINATION, utils.SETUP_TIME, utils.ANSWER_TIME, utils.USAGE, utils.COST); err != nil {
-		return err
-	} else {
-		cdreCfg.ContentFields = flds
+func (self *CdreConfig) loadFromJsonCfg(jsnCfg *CdreJsonCfg) error {
+	if jsnCfg == nil {
+		return nil
+	}
+	var err error
+	if jsnCfg.Cdr_format != nil {
+		self.CdrFormat = *jsnCfg.Cdr_format
+	}
+	if jsnCfg.Field_separator != nil && len(*jsnCfg.Field_separator) > 0 { // Make sure we got at least one character so we don't get panic here
+		sepStr := *jsnCfg.Field_separator
+		self.FieldSeparator = rune(sepStr[0])
+	}
+	if jsnCfg.Data_usage_multiply_factor != nil {
+		self.DataUsageMultiplyFactor = *jsnCfg.Data_usage_multiply_factor
+	}
+	if jsnCfg.Sms_usage_multiply_factor != nil {
+		self.SmsUsageMultiplyFactor = *jsnCfg.Sms_usage_multiply_factor
+	}
+	if jsnCfg.Generic_usage_multiply_factor != nil {
+		self.GenericUsageMultiplyFactor = *jsnCfg.Generic_usage_multiply_factor
+	}
+	if jsnCfg.Cost_multiply_factor != nil {
+		self.CostMultiplyFactor = *jsnCfg.Cost_multiply_factor
+	}
+	if jsnCfg.Cost_rounding_decimals != nil {
+		self.CostRoundingDecimals = *jsnCfg.Cost_rounding_decimals
+	}
+	if jsnCfg.Cost_shift_digits != nil {
+		self.CostShiftDigits = *jsnCfg.Cost_shift_digits
+	}
+	if jsnCfg.Mask_destination_id != nil {
+		self.MaskDestId = *jsnCfg.Mask_destination_id
+	}
+	if jsnCfg.Mask_length != nil {
+		self.MaskLength = *jsnCfg.Mask_length
+	}
+	if jsnCfg.Export_dir != nil {
+		self.ExportDir = *jsnCfg.Export_dir
+	}
+	if jsnCfg.Header_fields != nil {
+		if self.HeaderFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Header_fields); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Content_fields != nil {
+		if self.ContentFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Content_fields); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Trailer_fields != nil {
+		if self.TrailerFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Trailer_fields); err != nil {
+			return err
+		}
 	}
 	return nil
 }
 
-type CdreCdrField struct {
-	Name            string
-	Type            string
-	Value           string
-	Width           int
-	Strip           string
-	Padding         string
-	Layout          string
-	Filter          *utils.RSRField
-	Mandatory       bool
-	valueAsRsrField *utils.RSRField // Cached if the need arrises
-}
-
-func (cdrField *CdreCdrField) ValueAsRSRField() *utils.RSRField {
-	return cdrField.valueAsRsrField
-}
-
-// Should be called on .fwv configuration without providing default values for fixed with parameters
-func (cdrField *CdreCdrField) setDefaultFieldProperties(fixedWidth bool) error {
-	if cdrField.valueAsRsrField == nil {
-		return errors.New("Missing valueAsRsrField")
+// Clone itself into a new CdreConfig
+func (self *CdreConfig) Clone() *CdreConfig {
+	clnCdre := new(CdreConfig)
+	clnCdre.CdrFormat = self.CdrFormat
+	clnCdre.FieldSeparator = self.FieldSeparator
+	clnCdre.DataUsageMultiplyFactor = self.DataUsageMultiplyFactor
+	clnCdre.SmsUsageMultiplyFactor = self.SmsUsageMultiplyFactor
+	clnCdre.GenericUsageMultiplyFactor = self.GenericUsageMultiplyFactor
+	clnCdre.CostMultiplyFactor = self.CostMultiplyFactor
+	clnCdre.CostRoundingDecimals = self.CostRoundingDecimals
+	clnCdre.CostShiftDigits = self.CostShiftDigits
+	clnCdre.MaskDestId = self.MaskDestId
+	clnCdre.MaskLength = self.MaskLength
+	clnCdre.ExportDir = self.ExportDir
+	clnCdre.HeaderFields = make([]*CfgCdrField, len(self.HeaderFields))
+	for idx, fld := range self.HeaderFields {
+		clonedVal := *fld
+		clnCdre.HeaderFields[idx] = &clonedVal
 	}
-	switch cdrField.valueAsRsrField.Id {
-	case utils.CGRID:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 40
-		}
-	case utils.ORDERID:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 11
-			cdrField.Padding = "left"
-		}
-	case utils.TOR:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 6
-			cdrField.Padding = "left"
-		}
-	case utils.ACCID:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 36
-			cdrField.Strip = "left"
-			cdrField.Padding = "left"
-		}
-	case utils.CDRHOST:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 15
-			cdrField.Strip = "left"
-			cdrField.Padding = "left"
-		}
-	case utils.CDRSOURCE:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 15
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-		}
-	case utils.REQTYPE:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 13
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-		}
-	case utils.DIRECTION:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 4
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-		}
-	case utils.TENANT:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 24
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-		}
-	case utils.CATEGORY:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 10
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-		}
-	case utils.ACCOUNT:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 24
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-		}
-	case utils.SUBJECT:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 24
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-		}
-	case utils.DESTINATION:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 24
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-		}
-	case utils.SETUP_TIME:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 30
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-			cdrField.Layout = "2006-01-02T15:04:05Z07:00"
-		}
-	case utils.ANSWER_TIME:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 30
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-			cdrField.Layout = "2006-01-02T15:04:05Z07:00"
-		}
-	case utils.USAGE:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 30
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-		}
-	case utils.MEDI_RUNID:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 20
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-		}
-	case utils.COST:
-		cdrField.Mandatory = true
-		if fixedWidth {
-			cdrField.Width = 24
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-		}
-	default:
-		cdrField.Mandatory = false
-		if fixedWidth {
-			cdrField.Width = 30
-			cdrField.Strip = "xright"
-			cdrField.Padding = "left"
-		}
+	clnCdre.ContentFields = make([]*CfgCdrField, len(self.ContentFields))
+	for idx, fld := range self.ContentFields {
+		clonedVal := *fld
+		clnCdre.ContentFields[idx] = &clonedVal
 	}
-	return nil
+	clnCdre.TrailerFields = make([]*CfgCdrField, len(self.TrailerFields))
+	for idx, fld := range self.TrailerFields {
+		clonedVal := *fld
+		clnCdre.TrailerFields[idx] = &clonedVal
+	}
+	return clnCdre
 }

config/cdreconfig_test.go

@@ -1,6 +1,6 @@
 /*
 Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
+Copyright (C) ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -15,7 +15,6 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
-
 package config
 
 import (
@@ -24,460 +23,75 @@ import (
 	"testing"
 )
 
-func TestCdreCfgNewCdreCdrFieldsFromIds(t *testing.T) {
-	expectedFlds := []*CdreCdrField{
-		&CdreCdrField{
-			Name:            utils.CGRID,
-			Type:            utils.CDRFIELD,
-			Value:           utils.CGRID,
-			Width:           40,
-			Strip:           "",
-			Padding:         "",
-			Layout:          "",
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.CGRID},
-		},
-		&CdreCdrField{
-			Name:            "extra1",
-			Type:            utils.CDRFIELD,
-			Value:           "extra1",
-			Width:           30,
-			Strip:           "xright",
-			Padding:         "left",
-			Layout:          "",
-			Mandatory:       false,
-			valueAsRsrField: &utils.RSRField{Id: "extra1"},
-		},
+func TestCdreCfgClone(t *testing.T) {
+	cgrIdRsrs, _ := utils.ParseRSRFields("cgrid", utils.INFIELD_SEP)
+	runIdRsrs, _ := utils.ParseRSRFields("mediation_runid", utils.INFIELD_SEP)
+	emptyFields := []*CfgCdrField{}
+	initContentFlds := []*CfgCdrField{
+		&CfgCdrField{Tag: "CgrId",
+			Type:       "cdrfield",
+			CdrFieldId: "cgrid",
+			Value:      cgrIdRsrs},
+		&CfgCdrField{Tag: "RunId",
+			Type:       "cdrfield",
+			CdrFieldId: "mediation_runid",
+			Value:      runIdRsrs},
 	}
-	if cdreFlds, err := NewCdreCdrFieldsFromIds(true, utils.CGRID, "extra1"); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(expectedFlds, cdreFlds) {
-		t.Errorf("Expected: %v, received: %v", expectedFlds, cdreFlds)
+	initCdreCfg := &CdreConfig{
+		CdrFormat:               "csv",
+		FieldSeparator:          rune(','),
+		DataUsageMultiplyFactor: 1.0,
+		CostMultiplyFactor:      1.0,
+		CostRoundingDecimals:    -1,
+		CostShiftDigits:         0,
+		MaskDestId:              "MASKED_DESTINATIONS",
+		MaskLength:              0,
+		ExportDir:               "/var/log/cgrates/cdre",
+		ContentFields:           initContentFlds,
+	}
+	eClnContentFlds := []*CfgCdrField{
+		&CfgCdrField{Tag: "CgrId",
+			Type:       "cdrfield",
+			CdrFieldId: "cgrid",
+			Value:      cgrIdRsrs},
+		&CfgCdrField{Tag: "RunId",
+			Type:       "cdrfield",
+			CdrFieldId: "mediation_runid",
+			Value:      runIdRsrs},
+	}
+	eClnCdreCfg := &CdreConfig{
+		CdrFormat:               "csv",
+		FieldSeparator:          rune(','),
+		DataUsageMultiplyFactor: 1.0,
+		CostMultiplyFactor:      1.0,
+		CostRoundingDecimals:    -1,
+		CostShiftDigits:         0,
+		MaskDestId:              "MASKED_DESTINATIONS",
+		MaskLength:              0,
+		ExportDir:               "/var/log/cgrates/cdre",
+		HeaderFields:            emptyFields,
+		ContentFields:           eClnContentFlds,
+		TrailerFields:           emptyFields,
+	}
+	clnCdreCfg := initCdreCfg.Clone()
+	if !reflect.DeepEqual(eClnCdreCfg, clnCdreCfg) {
+		t.Errorf("Cloned result: %+v", clnCdreCfg)
+	}
+	initCdreCfg.DataUsageMultiplyFactor = 1024.0
+	if !reflect.DeepEqual(eClnCdreCfg, clnCdreCfg) { // MOdifying a field after clone should not affect cloned instance
+		t.Errorf("Cloned result: %+v", clnCdreCfg)
+	}
+	initContentFlds[0].Tag = "Destination"
+	if !reflect.DeepEqual(eClnCdreCfg, clnCdreCfg) { // MOdifying a field after clone should not affect cloned instance
+		t.Errorf("Cloned result: %+v", clnCdreCfg)
+	}
+	clnCdreCfg.CostShiftDigits = 2
+	if initCdreCfg.CostShiftDigits != 0 {
+		t.Error("Unexpected CostShiftDigits: ", initCdreCfg.CostShiftDigits)
+	}
+	clnCdreCfg.ContentFields[0].CdrFieldId = "destination"
+	if initCdreCfg.ContentFields[0].CdrFieldId != "cgrid" {
+		t.Error("Unexpected change of CdrFieldId: ", initCdreCfg.ContentFields[0].CdrFieldId)
 	}
-}
 
-func TestCdreCfgValueAsRSRField(t *testing.T) {
-	cdreCdrFld := &CdreCdrField{
-		Name:            utils.CGRID,
-		Type:            utils.CDRFIELD,
-		Value:           utils.CGRID,
-		Width:           10,
-		Strip:           "xright",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.CGRID},
-	}
-	if rsrVal := cdreCdrFld.ValueAsRSRField(); rsrVal != cdreCdrFld.valueAsRsrField {
-		t.Error("Unexpected value received: ", rsrVal)
-	}
-}
-
-func TestCdreCfgNewDefaultCdreConfig(t *testing.T) {
-	eCdreCfg := new(CdreConfig)
-	eCdreCfg.CdrFormat = utils.CSV
-	eCdreCfg.FieldSeparator = utils.CSV_SEP
-	eCdreCfg.DataUsageMultiplyFactor = 0.0
-	eCdreCfg.CostMultiplyFactor = 0.0
-	eCdreCfg.CostRoundingDecimals = -1
-	eCdreCfg.CostShiftDigits = 0
-	eCdreCfg.MaskDestId = ""
-	eCdreCfg.MaskLength = 0
-	eCdreCfg.ExportDir = "/var/log/cgrates/cdre"
-	eCdreCfg.ContentFields = []*CdreCdrField{
-		&CdreCdrField{
-			Name:            utils.CGRID,
-			Type:            utils.CDRFIELD,
-			Value:           utils.CGRID,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.CGRID},
-		},
-		&CdreCdrField{
-			Name:            utils.MEDI_RUNID,
-			Type:            utils.CDRFIELD,
-			Value:           utils.MEDI_RUNID,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.MEDI_RUNID},
-		},
-		&CdreCdrField{
-			Name:            utils.TOR,
-			Type:            utils.CDRFIELD,
-			Value:           utils.TOR,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.TOR},
-		},
-		&CdreCdrField{
-			Name:            utils.ACCID,
-			Type:            utils.CDRFIELD,
-			Value:           utils.ACCID,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.ACCID},
-		},
-		&CdreCdrField{
-			Name:            utils.REQTYPE,
-			Type:            utils.CDRFIELD,
-			Value:           utils.REQTYPE,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.REQTYPE},
-		},
-		&CdreCdrField{
-			Name:            utils.DIRECTION,
-			Type:            utils.CDRFIELD,
-			Value:           utils.DIRECTION,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.DIRECTION},
-		},
-		&CdreCdrField{
-			Name:            utils.TENANT,
-			Type:            utils.CDRFIELD,
-			Value:           utils.TENANT,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.TENANT},
-		},
-		&CdreCdrField{
-			Name:            utils.CATEGORY,
-			Type:            utils.CDRFIELD,
-			Value:           utils.CATEGORY,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.CATEGORY},
-		},
-		&CdreCdrField{
-			Name:            utils.ACCOUNT,
-			Type:            utils.CDRFIELD,
-			Value:           utils.ACCOUNT,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.ACCOUNT},
-		},
-		&CdreCdrField{
-			Name:            utils.SUBJECT,
-			Type:            utils.CDRFIELD,
-			Value:           utils.SUBJECT,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.SUBJECT},
-		},
-		&CdreCdrField{
-			Name:            utils.DESTINATION,
-			Type:            utils.CDRFIELD,
-			Value:           utils.DESTINATION,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.DESTINATION},
-		},
-		&CdreCdrField{
-			Name:            utils.SETUP_TIME,
-			Type:            utils.CDRFIELD,
-			Value:           utils.SETUP_TIME,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.SETUP_TIME},
-		},
-		&CdreCdrField{
-			Name:            utils.ANSWER_TIME,
-			Type:            utils.CDRFIELD,
-			Value:           utils.ANSWER_TIME,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.ANSWER_TIME},
-		},
-		&CdreCdrField{
-			Name:            utils.USAGE,
-			Type:            utils.CDRFIELD,
-			Value:           utils.USAGE,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.USAGE},
-		},
-		&CdreCdrField{
-			Name:            utils.COST,
-			Type:            utils.CDRFIELD,
-			Value:           utils.COST,
-			Mandatory:       true,
-			valueAsRsrField: &utils.RSRField{Id: utils.COST},
-		},
-	}
-	if cdreCfg, err := NewDefaultCdreConfig(); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCfg, cdreCfg) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCfg, cdreCfg)
-	}
-}
-
-func TestCdreCfgSetDefaultFieldProperties(t *testing.T) {
-	cdreCdrFld := &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.CGRID},
-	}
-	eCdreCdrFld := &CdreCdrField{
-		Width:           40,
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.CGRID},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.ORDERID},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           11,
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.ORDERID},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.TOR},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           6,
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.TOR},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.ACCID},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           36,
-		Strip:           "left",
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.ACCID},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.CDRHOST},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           15,
-		Strip:           "left",
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.CDRHOST},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.CDRSOURCE},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           15,
-		Strip:           "xright",
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.CDRSOURCE},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.REQTYPE},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           13,
-		Strip:           "xright",
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.REQTYPE},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.DIRECTION},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           4,
-		Strip:           "xright",
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.DIRECTION},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.TENANT},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           24,
-		Strip:           "xright",
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.TENANT},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.CATEGORY},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           10,
-		Strip:           "xright",
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.CATEGORY},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.ACCOUNT},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           24,
-		Strip:           "xright",
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.ACCOUNT},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.SUBJECT},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           24,
-		Strip:           "xright",
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.SUBJECT},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.DESTINATION},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           24,
-		Strip:           "xright",
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.DESTINATION},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.SETUP_TIME},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           30,
-		Strip:           "xright",
-		Padding:         "left",
-		Layout:          "2006-01-02T15:04:05Z07:00",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.SETUP_TIME},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.ANSWER_TIME},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           30,
-		Strip:           "xright",
-		Padding:         "left",
-		Layout:          "2006-01-02T15:04:05Z07:00",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.ANSWER_TIME},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.USAGE},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           30,
-		Strip:           "xright",
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.USAGE},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.MEDI_RUNID},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           20,
-		Strip:           "xright",
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.MEDI_RUNID},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: utils.COST},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           24,
-		Strip:           "xright",
-		Padding:         "left",
-		Mandatory:       true,
-		valueAsRsrField: &utils.RSRField{Id: utils.COST},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
-	cdreCdrFld = &CdreCdrField{
-		valueAsRsrField: &utils.RSRField{Id: "extra_1"},
-	}
-	eCdreCdrFld = &CdreCdrField{
-		Width:           30,
-		Strip:           "xright",
-		Padding:         "left",
-		Mandatory:       false,
-		valueAsRsrField: &utils.RSRField{Id: "extra_1"},
-	}
-	if err := cdreCdrFld.setDefaultFieldProperties(true); err != nil {
-		t.Error(err)
-	} else if !reflect.DeepEqual(eCdreCdrFld, cdreCdrFld) {
-		t.Errorf("Expecting: %v, received: %v", eCdreCdrFld, cdreCdrFld)
-	}
 }

config/cdrstatsconfig.go

@@ -19,169 +19,16 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package config
 
 import (
-	"code.google.com/p/goconf/conf"
-	"github.com/cgrates/cgrates/utils"
-	"strconv"
 	"time"
 )
 
-// Parse the configuration file for CDRStatConfigs
-func ParseCfgDefaultCDRStatsConfig(c *conf.ConfigFile) (*CdrStatsConfig, error) {
-	var err error
-	csCfg := NewCdrStatsConfigWithDefaults()
-	if hasOpt := c.HasOption("cdrstats", "queue_length"); hasOpt {
-		csCfg.QueueLength, _ = c.GetInt("cdrstats", "queue_length")
-	}
-	if hasOpt := c.HasOption("cdrstats", "time_window"); hasOpt {
-		durStr, _ := c.GetString("cdrstats", "time_window")
-		if csCfg.TimeWindow, err = utils.ParseDurationWithSecs(durStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "metrics"); hasOpt {
-		metricsStr, _ := c.GetString("cdrstats", "metrics")
-		if csCfg.Metrics, err = ConfigSlice(metricsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "setup_interval"); hasOpt {
-		setupIntervalStr, _ := c.GetString("cdrstats", "setup_interval")
-		if len(setupIntervalStr) != 0 { // If we parse empty, will get empty time, we prefer nil
-			if setupIntervalSlc, err := ConfigSlice(setupIntervalStr); err != nil {
-				return nil, err
-			} else {
-				for _, setupTimeStr := range setupIntervalSlc {
-					if setupTime, err := utils.ParseTimeDetectLayout(setupTimeStr); err != nil {
-						return nil, err
-					} else {
-						csCfg.SetupInterval = append(csCfg.SetupInterval, setupTime)
-					}
-				}
-			}
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "tors"); hasOpt {
-		torsStr, _ := c.GetString("cdrstats", "tors")
-		if csCfg.TORs, err = ConfigSlice(torsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "cdr_hosts"); hasOpt {
-		valsStr, _ := c.GetString("cdrstats", "cdr_hosts")
-		if csCfg.CdrHosts, err = ConfigSlice(valsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "cdr_sources"); hasOpt {
-		valsStr, _ := c.GetString("cdrstats", "cdr_sources")
-		if csCfg.CdrSources, err = ConfigSlice(valsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "req_types"); hasOpt {
-		valsStr, _ := c.GetString("cdrstats", "req_types")
-		if csCfg.ReqTypes, err = ConfigSlice(valsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "directions"); hasOpt {
-		valsStr, _ := c.GetString("cdrstats", "directions")
-		if csCfg.Directions, err = ConfigSlice(valsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "tenants"); hasOpt {
-		valsStr, _ := c.GetString("cdrstats", "tenants")
-		if csCfg.Tenants, err = ConfigSlice(valsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "categories"); hasOpt {
-		valsStr, _ := c.GetString("cdrstats", "categories")
-		if csCfg.Categories, err = ConfigSlice(valsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "accounts"); hasOpt {
-		valsStr, _ := c.GetString("cdrstats", "accounts")
-		if csCfg.Accounts, err = ConfigSlice(valsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "subjects"); hasOpt {
-		valsStr, _ := c.GetString("cdrstats", "subjects")
-		if csCfg.Subjects, err = ConfigSlice(valsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "destination_prefixes"); hasOpt {
-		valsStr, _ := c.GetString("cdrstats", "destination_prefixes")
-		if csCfg.DestinationPrefixes, err = ConfigSlice(valsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "usage_interval"); hasOpt {
-		usageIntervalStr, _ := c.GetString("cdrstats", "usage_interval")
-		if usageIntervalSlc, err := ConfigSlice(usageIntervalStr); err != nil {
-			return nil, err
-		} else {
-			for _, usageDurStr := range usageIntervalSlc {
-				if usageDur, err := utils.ParseDurationWithSecs(usageDurStr); err != nil {
-					return nil, err
-				} else {
-					csCfg.UsageInterval = append(csCfg.UsageInterval, usageDur)
-				}
-			}
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "mediation_run_ids"); hasOpt {
-		valsStr, _ := c.GetString("cdrstats", "mediation_run_ids")
-		if csCfg.MediationRunIds, err = ConfigSlice(valsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "rated_accounts"); hasOpt {
-		valsStr, _ := c.GetString("cdrstats", "rated_accounts")
-		if csCfg.RatedAccounts, err = ConfigSlice(valsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "rated_subjects"); hasOpt {
-		valsStr, _ := c.GetString("cdrstats", "rated_subjects")
-		if csCfg.RatedSubjects, err = ConfigSlice(valsStr); err != nil {
-			return nil, err
-		}
-	}
-	if hasOpt := c.HasOption("cdrstats", "cost_intervals"); hasOpt {
-		valsStr, _ := c.GetString("cdrstats", "cost_intervals")
-		if costSlc, err := ConfigSlice(valsStr); err != nil {
-			return nil, err
-		} else {
-			for _, costStr := range costSlc {
-				if cost, err := strconv.ParseFloat(costStr, 64); err != nil {
-					return nil, err
-				} else {
-					csCfg.CostInterval = append(csCfg.CostInterval, cost)
-				}
-			}
-		}
-	}
-
-	return csCfg, nil
-}
-
-func NewCdrStatsConfigWithDefaults() *CdrStatsConfig {
-	csCfg := new(CdrStatsConfig)
-	csCfg.setDefaults()
-	return csCfg
-}
-
 type CdrStatsConfig struct {
 	Id                  string        // Config id, unique per config instance
 	QueueLength         int           // Number of items in the stats buffer
 	TimeWindow          time.Duration // Will only keep the CDRs who's call setup time is not older than time.Now()-TimeWindow
-	Metrics             []string      // ASR, ACD, ACC
-	SetupInterval       []time.Time   // 2 or less items (>= start interval,< stop_interval)
+	SaveInterval        time.Duration
+	Metrics             []string    // ASR, ACD, ACC
+	SetupInterval       []time.Time // 2 or less items (>= start interval,< stop_interval)
 	TORs                []string
 	CdrHosts            []string
 	CdrSources          []string
@@ -193,15 +40,10 @@ type CdrStatsConfig struct {
 	Subjects            []string
 	DestinationPrefixes []string
 	UsageInterval       []time.Duration // 2 or less items (>= Usage, <Usage)
+	Suppliers           []string
+	DisconnectCauses    []string
 	MediationRunIds     []string
 	RatedAccounts       []string
 	RatedSubjects       []string
 	CostInterval        []float64 // 2 or less items, (>=Cost, <Cost)
 }
-
-func (csCfg *CdrStatsConfig) setDefaults() {
-	csCfg.Id = utils.META_DEFAULT
-	csCfg.QueueLength = 50
-	csCfg.TimeWindow = time.Duration(1) * time.Hour
-	csCfg.Metrics = []string{"ASR", "ACD", "ACC"}
-}

config/cfg_data.json

@@ -0,0 +1,51 @@
+{
+
+// Real-time Charging System for Telecom & ISP environments
+// Copyright (C) ITsysCOM GmbH
+//
+// This file contains the default configuration hardcoded into CGRateS.
+// This is what you get when you load CGRateS with an empty configuration file.
+
+"general": {
+	"default_reqtype": "*pseudoprepaid",				// default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated>
+},
+
+"cdrs": {
+	"enabled": true,						// start the CDR Server service:  <true|false>
+},
+
+"rater": {
+	"enabled": true,						// enable Rater service: <true|false>
+},
+
+"cdrc": {
+	"CDRC-CSV1": {
+		"enabled": true,							// enable CDR client functionality
+		"cdr_in_dir": "/tmp/cgrates/cdrc1/in",		// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/tmp/cgrates/cdrc1/out",	// absolute path towards the directory where processed CDRs will be moved
+		"cdr_source_id": "csv1",					// free form field, tag identifying the source of the CDRs within CDRS database
+	},
+	"CDRC-CSV2": {
+		"enabled": true,							// enable CDR client functionality
+		"cdr_in_dir": "/tmp/cgrates/cdrc2/in",		// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/tmp/cgrates/cdrc2/out",	// absolute path towards the directory where processed CDRs will be moved
+		"data_usage_multiply_factor": 0.000976563,
+		"run_delay": 1,
+		"cdr_source_id": "csv2",					// free form field, tag identifying the source of the CDRs within CDRS database
+		"content_fields":[							// import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
+			{"cdr_field_id": "tor", "value": "~7:s/^(voice|data|sms|generic)$/*$1/"},
+			{"cdr_field_id": "answer_time", "value": "1"},
+			{"cdr_field_id": "usage", "value": "~9:s/^(\\d+)$/${1}s/"},
+		],
+	},
+},
+
+"sm_freeswitch": {
+	"enabled": true,				// starts SessionManager service: <true|false>
+	"connections":[					// instantiate connections to multiple FreeSWITCH servers
+		{"server": "1.2.3.4:8021", "password": "ClueCon", "reconnects": 5},
+		{"server": "2.3.4.5:8021", "password": "ClueCon", "reconnects": 5},
+	],
+},
+
+}

config/cfg_data2.json

@@ -0,0 +1,30 @@
+{
+
+"cdrc": {
+	"CDRC-CSV2": {
+		"enabled": true,							// enable CDR client functionality
+		"cdr_in_dir": "/tmp/cgrates/cdrc2/in",		// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/tmp/cgrates/cdrc2/out",	// absolute path towards the directory where processed CDRs will be moved
+		"data_usage_multiply_factor": 0.000976563,
+		"cdr_source_id": "csv2",					// free form field, tag identifying the source of the CDRs within CDRS database
+		"content_fields":[								// import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
+			{"cdr_field_id": "tor", "value": "~7:s/^(voice|data|sms|generic)$/*$1/"},
+			{"cdr_field_id": "answer_time", "value": "2"},
+		],
+	},
+	"CDRC-CSV3": {
+		"enabled": true,							// enable CDR client functionality
+		"cdr_in_dir": "/tmp/cgrates/cdrc3/in",		// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/tmp/cgrates/cdrc3/out",	// absolute path towards the directory where processed CDRs will be moved
+		"cdr_source_id": "csv3",					// free form field, tag identifying the source of the CDRs within CDRS database
+	},
+},
+
+"sm_freeswitch": {
+	"enabled": true,				// starts SessionManager service: <true|false>
+	"connections":[					// instantiate connections to multiple FreeSWITCH servers
+		{"server": "2.3.4.5:8021", "password": "ClueCon", "reconnects": 5},
+	],
+},
+
+}

config/cfgcdrfield.go

@@ -0,0 +1,92 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"github.com/cgrates/cgrates/utils"
+)
+
+func NewCfgCdrFieldFromCdrFieldJsonCfg(jsnCfgFld *CdrFieldJsonCfg) (*CfgCdrField, error) {
+	var err error
+	cfgFld := new(CfgCdrField)
+	if jsnCfgFld.Tag != nil {
+		cfgFld.Tag = *jsnCfgFld.Tag
+	}
+	if jsnCfgFld.Type != nil {
+		cfgFld.Type = *jsnCfgFld.Type
+	}
+	if jsnCfgFld.Cdr_field_id != nil {
+		cfgFld.CdrFieldId = *jsnCfgFld.Cdr_field_id
+	}
+	if jsnCfgFld.Metatag_id != nil {
+		cfgFld.MetatagId = *jsnCfgFld.Metatag_id
+	}
+	if jsnCfgFld.Value != nil {
+		if cfgFld.Value, err = utils.ParseRSRFields(*jsnCfgFld.Value, utils.INFIELD_SEP); err != nil {
+			return nil, err
+		}
+	}
+	if jsnCfgFld.Field_filter != nil {
+		if cfgFld.FieldFilter, err = utils.ParseRSRFields(*jsnCfgFld.Field_filter, utils.INFIELD_SEP); err != nil {
+			return nil, err
+		}
+	}
+	if jsnCfgFld.Width != nil {
+		cfgFld.Width = *jsnCfgFld.Width
+	}
+	if jsnCfgFld.Strip != nil {
+		cfgFld.Strip = *jsnCfgFld.Strip
+	}
+	if jsnCfgFld.Padding != nil {
+		cfgFld.Padding = *jsnCfgFld.Padding
+	}
+	if jsnCfgFld.Layout != nil {
+		cfgFld.Layout = *jsnCfgFld.Layout
+	}
+	if jsnCfgFld.Mandatory != nil {
+		cfgFld.Mandatory = *jsnCfgFld.Mandatory
+	}
+	return cfgFld, nil
+}
+
+type CfgCdrField struct {
+	Tag         string // Identifier for the administrator
+	Type        string // Type of field
+	CdrFieldId  string // StoredCdr field name
+	MetatagId   string
+	Value       utils.RSRFields
+	FieldFilter utils.RSRFields
+	Width       int
+	Strip       string
+	Padding     string
+	Layout      string
+	Mandatory   bool
+}
+
+func CfgCdrFieldsFromCdrFieldsJsonCfg(jsnCfgFldss []*CdrFieldJsonCfg) ([]*CfgCdrField, error) {
+	retFields := make([]*CfgCdrField, len(jsnCfgFldss))
+	for idx, jsnFld := range jsnCfgFldss {
+		if cfgFld, err := NewCfgCdrFieldFromCdrFieldJsonCfg(jsnFld); err != nil {
+			return nil, err
+		} else {
+			retFields[idx] = cfgFld
+		}
+	}
+	return retFields, nil
+}

config/cfgcdrfield_test.go

@@ -0,0 +1,19 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config

config/config_defaults.go

@@ -0,0 +1,270 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+const CGRATES_CFG_JSON = `
+{
+
+// Real-time Charging System for Telecom & ISP environments
+// Copyright (C) ITsysCOM GmbH
+//
+// This file contains the default configuration hardcoded into CGRateS.
+// This is what you get when you load CGRateS with an empty configuration file.
+
+"general": {
+	"http_skip_tls_verify": false,			// if enabled Http Client will accept any TLS certificate
+	"rounding_decimals": 10,				// system level precision for floats
+	"dbdata_encoding": "msgpack",			// encoding used to store object data in strings: <msgpack|json>
+	"tpexport_dir": "/var/log/cgrates/tpe",	// path towards export folder for offline Tariff Plans
+	"default_reqtype": "*rated",			// default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated>
+	"default_category": "call",				// default Type of Record to consider when missing from requests
+	"default_tenant": "cgrates.org",		// default Tenant to consider when missing from requests
+	"default_subject": "cgrates",			// default rating Subject to consider when missing from requests
+	"connect_attempts": 3,                  // initial server connect attempts
+	"reconnects": -1,                       // number of retries in case of connection lost
+},
+
+
+"listen": {
+	"rpc_json": "127.0.0.1:2012",			// RPC JSON listening address
+	"rpc_gob": "127.0.0.1:2013",			// RPC GOB listening address
+	"http": "127.0.0.1:2080",				// HTTP listening address
+},
+
+
+"tariffplan_db": {							// database used to store active tariff plan configuration
+	"db_type": "redis",						// tariffplan_db type: <redis>
+	"db_host": "127.0.0.1",					// tariffplan_db host address
+	"db_port": 6379, 						// port to reach the tariffplan_db
+	"db_name": "10", 						// tariffplan_db name to connect to
+	"db_user": "", 							// sername to use when connecting to tariffplan_db
+	"db_passwd": "", 						// password to use when connecting to tariffplan_db
+},
+
+
+"data_db": {								// database used to store runtime data (eg: accounts, cdr stats)
+	"db_type": "redis",						// data_db type: <redis>
+	"db_host": "127.0.0.1",					// data_db host address
+	"db_port": 6379, 						// data_db port to reach the database
+	"db_name": "11", 						// data_db database name to connect to
+	"db_user": "", 							// username to use when connecting to data_db
+	"db_passwd": "", 						// password to use when connecting to data_db
+},
+
+
+"stor_db": {								// database used to store offline tariff plans and CDRs
+	"db_type": "mysql",						// stor database type to use: <mysql|postgres>
+	"db_host": "127.0.0.1",					// the host to connect to
+	"db_port": 3306, 						// the port to reach the stordb
+	"db_name": "cgrates", 					// stor database name
+	"db_user": "cgrates", 					// username to use when connecting to stordb
+	"db_passwd": "CGRateS.org", 			// password to use when connecting to stordb
+	"max_open_conns": 100,					// maximum database connections opened
+	"max_idle_conns": 10,					// maximum database connections idle
+},
+
+
+"balancer": {
+	"enabled": false, 						// start Balancer service: <true|false>
+},
+
+
+"rater": {
+	"enabled": false,						// enable Rater service: <true|false>
+	"balancer": "",							// register to balancer as worker: <""|internal|x.y.z.y:1234>
+	"cdrstats": "",							// address where to reach the cdrstats service, empty to disable stats functionality: <""|internal|x.y.z.y:1234>
+	"historys": "",							// address where to reach the history service, empty to disable history functionality: <""|internal|x.y.z.y:1234>
+	"pubsubs": "",							// address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234>
+	"users": "",							// address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234>
+},
+
+
+"scheduler": {
+	"enabled": false,						// start Scheduler service: <true|false>
+},
+
+
+"cdrs": {
+	"enabled": false,						// start the CDR Server service:  <true|false>
+	"extra_fields": [],						// extra fields to store in CDRs for non-generic CDRs
+	"store_cdrs": true,						// store cdrs in storDb
+	"rater": "internal",					// address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234>
+	"cdrstats": "",							// address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234>
+	"reconnects": 5,						// number of reconnect attempts to rater or cdrs
+	"cdr_replication":[],					// replicate the raw CDR to a number of servers
+},
+
+
+"cdrstats": {
+	"enabled": false,						// starts the cdrstats service: <true|false>
+	"save_interval": "1m",					// interval to save changed stats into dataDb storage
+},
+
+
+"cdre": {
+	"*default": {
+		"cdr_format": "csv",							// exported CDRs format <csv>
+		"field_separator": ",",
+		"data_usage_multiply_factor": 1,				// multiply data usage before export (eg: convert from KBytes to Bytes)
+		"sms_usage_multiply_factor": 1,					// multiply data usage before export (eg: convert from SMS unit to call duration in some billing systems)
+        "generic_usage_multiply_factor": 1,					// multiply data usage before export (eg: convert from GENERIC unit to call duration in some billing systems)
+		"cost_multiply_factor": 1,						// multiply cost before export, eg: add VAT
+		"cost_rounding_decimals": -1,					// rounding decimals for Cost values. -1 to disable rounding
+		"cost_shift_digits": 0,							// shift digits in the cost on export (eg: convert from EUR to cents)
+		"mask_destination_id": "MASKED_DESTINATIONS",	// destination id containing called addresses to be masked on export
+		"mask_length": 0,								// length of the destination suffix to be masked
+		"export_dir": "/var/log/cgrates/cdre",			// path where the exported CDRs will be placed
+		"header_fields": [],							// template of the exported header fields
+		"content_fields": [								// template of the exported content fields
+			{"tag": "CgrId", "cdr_field_id": "cgrid", "type": "cdrfield", "value": "cgrid"},
+			{"tag":"RunId", "cdr_field_id": "mediation_runid", "type": "cdrfield", "value": "mediation_runid"},
+			{"tag":"Tor", "cdr_field_id": "tor", "type": "cdrfield", "value": "tor"},
+			{"tag":"AccId", "cdr_field_id": "accid", "type": "cdrfield", "value": "accid"},
+			{"tag":"ReqType", "cdr_field_id": "reqtype", "type": "cdrfield", "value": "reqtype"},
+			{"tag":"Direction", "cdr_field_id": "direction", "type": "cdrfield", "value": "direction"},
+			{"tag":"Tenant", "cdr_field_id": "tenant", "type": "cdrfield", "value": "tenant"},
+			{"tag":"Category", "cdr_field_id": "category", "type": "cdrfield", "value": "category"},
+			{"tag":"Account", "cdr_field_id": "account", "type": "cdrfield", "value": "account"},
+			{"tag":"Subject", "cdr_field_id": "subject", "type": "cdrfield", "value": "subject"},
+			{"tag":"Destination", "cdr_field_id": "destination", "type": "cdrfield", "value": "destination"},
+			{"tag":"SetupTime", "cdr_field_id": "setup_time", "type": "cdrfield", "value": "setup_time", "layout": "2006-01-02T15:04:05Z07:00"},
+			{"tag":"AnswerTime", "cdr_field_id": "answer_time", "type": "cdrfield", "value": "answer_time", "layout": "2006-01-02T15:04:05Z07:00"},
+			{"tag":"Usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "usage"},
+			{"tag":"Cost", "cdr_field_id": "cost", "type": "cdrfield", "value": "cost"},
+		],
+		"trailer_fields": [],							// template of the exported trailer fields
+	}
+},
+
+
+"cdrc": {
+	"*default": {
+		"enabled": false,							// enable CDR client functionality
+		"dry_run": false,							// do not send the CDRs to CDRS, just parse them
+		"cdrs": "internal",							// address where to reach CDR server. <internal|x.y.z.y:1234>
+		"cdr_format": "csv",						// CDR file format <csv|freeswitch_csv|fwv|opensips_flatstore>
+		"field_separator": ",",						// separator used in case of csv files
+		"run_delay": 0,								// sleep interval in seconds between consecutive runs, 0 to use automation via inotify
+		"max_open_files": 1024,						// maximum simultaneous files to process, 0 for unlimited
+		"data_usage_multiply_factor": 1024,			// conversion factor for data usage
+		"cdr_in_dir": "/var/log/cgrates/cdrc/in",	// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/var/log/cgrates/cdrc/out",	// absolute path towards the directory where processed CDRs will be moved
+		"failed_calls_prefix": "missed_calls",		// used in case of flatstore CDRs to avoid searching for BYE records
+		"cdr_source_id": "freeswitch_csv",			// free form field, tag identifying the source of the CDRs within CDRS database
+		"cdr_filter": "",							// filter CDR records to import
+		"partial_record_cache": "10s",				// duration to cache partial records when not pairing
+		"header_fields": [],						// template of the import header fields
+		"content_fields":[							// import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
+			{"tag": "tor", "cdr_field_id": "tor", "type": "cdrfield", "value": "2", "mandatory": true},
+			{"tag": "accid", "cdr_field_id": "accid", "type": "cdrfield", "value": "3", "mandatory": true},
+			{"tag": "reqtype", "cdr_field_id": "reqtype", "type": "cdrfield", "value": "4", "mandatory": true},
+			{"tag": "direction", "cdr_field_id": "direction", "type": "cdrfield", "value": "5", "mandatory": true},
+			{"tag": "tenant", "cdr_field_id": "tenant", "type": "cdrfield", "value": "6", "mandatory": true},
+			{"tag": "category", "cdr_field_id": "category", "type": "cdrfield", "value": "7", "mandatory": true},
+			{"tag": "account", "cdr_field_id": "account", "type": "cdrfield", "value": "8", "mandatory": true},
+			{"tag": "subject", "cdr_field_id": "subject", "type": "cdrfield", "value": "9", "mandatory": true},
+			{"tag": "destination", "cdr_field_id": "destination", "type": "cdrfield", "value": "10", "mandatory": true},
+			{"tag": "setup_time", "cdr_field_id": "setup_time", "type": "cdrfield", "value": "11", "mandatory": true},
+			{"tag": "answer_time", "cdr_field_id": "answer_time", "type": "cdrfield", "value": "12", "mandatory": true},
+			{"tag": "usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "13", "mandatory": true},
+		],
+		"trailer_fields": [],							// template of the import trailer fields
+	}
+},
+
+
+"sm_freeswitch": {
+	"enabled": false,				// starts SessionManager service: <true|false>
+	"rater": "internal",			// address where to reach the Rater <""|internal|127.0.0.1:2013>
+	"cdrs": "internal",				// address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234>
+	"reconnects": 5,				// number of reconnect attempts to rater or cdrs
+	"create_cdr": false,			// create CDR out of events and sends them to CDRS component
+	"cdr_extra_fields": [],			// extra fields to store in CDRs when creating them
+	"debit_interval": "10s",		// interval to perform debits on.
+	"min_call_duration": "0s",		// only authorize calls with allowed duration higher than this
+	"max_call_duration": "3h",		// maximum call duration a prepaid call can last
+	"min_dur_low_balance": "5s",	// threshold which will trigger low balance warnings for prepaid calls (needs to be lower than debit_interval)
+	"low_balance_ann_file": "",		// file to be played when low balance is reached for prepaid calls
+	"empty_balance_context": "",	// if defined, prepaid calls will be transfered to this context on empty balance
+	"empty_balance_ann_file": "",	// file to be played before disconnecting prepaid calls on empty balance (applies only if no context defined)
+	"subscribe_park": true,			// subscribe via fsock to receive park events
+	"channel_sync_interval": "5m",	// sync channels with freeswitch regularly
+	"connections":[					// instantiate connections to multiple FreeSWITCH servers
+		{"server": "127.0.0.1:8021", "password": "ClueCon", "reconnects": 5}
+	],
+},
+
+
+"sm_kamailio": {
+	"enabled": false,				// starts SessionManager service: <true|false>
+	"rater": "internal",			// address where to reach the Rater <""|internal|127.0.0.1:2013>
+	"cdrs": "internal",				// address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234>
+	"reconnects": 5,				// number of reconnect attempts to rater or cdrs
+	"create_cdr": false,			// create CDR out of events and sends them to CDRS component
+	"debit_interval": "10s",		// interval to perform debits on.
+	"min_call_duration": "0s",		// only authorize calls with allowed duration higher than this
+	"max_call_duration": "3h",		// maximum call duration a prepaid call can last
+	"connections":[					// instantiate connections to multiple Kamailio servers
+		{"evapi_addr": "127.0.0.1:8448", "reconnects": 5}
+	],
+},
+
+
+"sm_opensips": {
+	"enabled": false,					// starts SessionManager service: <true|false>
+	"listen_udp": "127.0.0.1:2020",		// address where to listen for datagram events coming from OpenSIPS
+	"rater": "internal",				// address where to reach the Rater <""|internal|127.0.0.1:2013>
+	"cdrs": "internal",					// address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234>
+	"reconnects": 5,					// number of reconnects if connection is lost
+	"create_cdr": false,				// create CDR out of events and sends them to CDRS component
+	"debit_interval": "10s",			// interval to perform debits on.
+	"min_call_duration": "0s",			// only authorize calls with allowed duration higher than this
+	"max_call_duration": "3h",			// maximum call duration a prepaid call can last
+	"events_subscribe_interval": "60s",	// automatic events subscription to OpenSIPS, 0 to disable it
+	"mi_addr": "127.0.0.1:8020",		// address where to reach OpenSIPS MI to send session disconnects
+},
+
+
+"historys": {
+	"enabled": false,							// starts History service: <true|false>.
+	"history_dir": "/var/log/cgrates/history",	// location on disk where to store history files.
+	"save_interval": "1s",						// interval to save changed cache into .git archive
+},
+
+
+"pubsubs": {
+	"enabled": false,							// starts PubSub service: <true|false>.
+},
+
+
+"users": {
+	"enabled": false,							// starts User service: <true|false>.
+	"indexes": [],                  			// user profile field indexes
+},
+
+
+"mailer": {
+	"server": "localhost",								// the server to use when sending emails out
+	"auth_user": "cgrates",								// authenticate to email server using this user
+	"auth_passwd": "CGRateS.org",						// authenticate to email server with this password
+	"from_address": "cgr-mailer@localhost.localdomain"	// from address used when sending emails out
+},
+
+
+}`

config/config_json.go

@@ -0,0 +1,281 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+
+	"github.com/DisposaBoy/JsonConfigReader"
+)
+
+const (
+	GENERAL_JSN    = "general"
+	LISTEN_JSN     = "listen"
+	TPDB_JSN       = "tariffplan_db"
+	DATADB_JSN     = "data_db"
+	STORDB_JSN     = "stor_db"
+	BALANCER_JSN   = "balancer"
+	RATER_JSN      = "rater"
+	SCHEDULER_JSN  = "scheduler"
+	CDRS_JSN       = "cdrs"
+	MEDIATOR_JSN   = "mediator"
+	CDRSTATS_JSN   = "cdrstats"
+	CDRE_JSN       = "cdre"
+	CDRC_JSN       = "cdrc"
+	SMFS_JSN       = "sm_freeswitch"
+	SMKAM_JSN      = "sm_kamailio"
+	SMOSIPS_JSN    = "sm_opensips"
+	SM_JSN         = "session_manager"
+	FS_JSN         = "freeswitch"
+	KAMAILIO_JSN   = "kamailio"
+	OSIPS_JSN      = "opensips"
+	HISTSERV_JSN   = "historys"
+	PUBSUBSERV_JSN = "pubsubs"
+	USERSERV_JSN   = "users"
+	MAILER_JSN     = "mailer"
+)
+
+// Loads the json config out of io.Reader, eg other sources than file, maybe over http
+func NewCgrJsonCfgFromReader(r io.Reader) (*CgrJsonCfg, error) {
+	var cgrJsonCfg CgrJsonCfg
+	jr := JsonConfigReader.New(r)
+	if err := json.NewDecoder(jr).Decode(&cgrJsonCfg); err != nil {
+		return nil, err
+	}
+	return &cgrJsonCfg, nil
+}
+
+// Loads the config out of file
+func NewCgrJsonCfgFromFile(fpath string) (*CgrJsonCfg, error) {
+	cfgFile, err := os.Open(fpath)
+	if err != nil {
+		return nil, err
+	}
+	defer cfgFile.Close()
+	return NewCgrJsonCfgFromReader(cfgFile)
+}
+
+// Main object holding the loaded config as section raw messages
+type CgrJsonCfg map[string]*json.RawMessage
+
+func (self CgrJsonCfg) GeneralJsonCfg() (*GeneralJsonCfg, error) {
+	rawCfg, hasKey := self[GENERAL_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(GeneralJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) ListenJsonCfg() (*ListenJsonCfg, error) {
+	rawCfg, hasKey := self[LISTEN_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(ListenJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) DbJsonCfg(section string) (*DbJsonCfg, error) {
+	rawCfg, hasKey := self[section]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(DbJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) BalancerJsonCfg() (*BalancerJsonCfg, error) {
+	rawCfg, hasKey := self[BALANCER_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(BalancerJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) RaterJsonCfg() (*RaterJsonCfg, error) {
+	rawCfg, hasKey := self[RATER_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(RaterJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) SchedulerJsonCfg() (*SchedulerJsonCfg, error) {
+	rawCfg, hasKey := self[SCHEDULER_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(SchedulerJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) CdrsJsonCfg() (*CdrsJsonCfg, error) {
+	rawCfg, hasKey := self[CDRS_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(CdrsJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) CdrStatsJsonCfg() (*CdrStatsJsonCfg, error) {
+	rawCfg, hasKey := self[CDRSTATS_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(CdrStatsJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) CdreJsonCfgs() (map[string]*CdreJsonCfg, error) {
+	rawCfg, hasKey := self[CDRE_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := make(map[string]*CdreJsonCfg)
+	if err := json.Unmarshal(*rawCfg, &cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) CdrcJsonCfg() (map[string]*CdrcJsonCfg, error) {
+	rawCfg, hasKey := self[CDRC_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := make(map[string]*CdrcJsonCfg)
+	if err := json.Unmarshal(*rawCfg, &cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) SmFsJsonCfg() (*SmFsJsonCfg, error) {
+	rawCfg, hasKey := self[SMFS_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(SmFsJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) SmKamJsonCfg() (*SmKamJsonCfg, error) {
+	rawCfg, hasKey := self[SMKAM_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(SmKamJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) SmOsipsJsonCfg() (*SmOsipsJsonCfg, error) {
+	rawCfg, hasKey := self[SMOSIPS_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(SmOsipsJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) HistServJsonCfg() (*HistServJsonCfg, error) {
+	rawCfg, hasKey := self[HISTSERV_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(HistServJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) PubSubServJsonCfg() (*PubSubServJsonCfg, error) {
+	rawCfg, hasKey := self[PUBSUBSERV_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(PubSubServJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) UserServJsonCfg() (*UserServJsonCfg, error) {
+	rawCfg, hasKey := self[USERSERV_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(UserServJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) MailerJsonCfg() (*MailerJsonCfg, error) {
+	rawCfg, hasKey := self[MAILER_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(MailerJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}

config/config_json_test.go

@@ -0,0 +1,503 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+var dfCgrJsonCfg *CgrJsonCfg
+
+// Loads up the default configuration and  tests it's sections one by one
+func TestDfNewdfCgrJsonCfgFromReader(t *testing.T) {
+	var err error
+	if dfCgrJsonCfg, err = NewCgrJsonCfgFromReader(strings.NewReader(CGRATES_CFG_JSON)); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestDfGeneralJsonCfg(t *testing.T) {
+	eCfg := &GeneralJsonCfg{
+		Http_skip_tls_verify: utils.BoolPointer(false),
+		Rounding_decimals:    utils.IntPointer(10),
+		Dbdata_encoding:      utils.StringPointer("msgpack"),
+		Tpexport_dir:         utils.StringPointer("/var/log/cgrates/tpe"),
+		Default_reqtype:      utils.StringPointer(utils.META_RATED),
+		Default_category:     utils.StringPointer("call"),
+		Default_tenant:       utils.StringPointer("cgrates.org"),
+		Default_subject:      utils.StringPointer("cgrates"),
+		Connect_attempts:     utils.IntPointer(3),
+		Reconnects:           utils.IntPointer(-1)}
+	if gCfg, err := dfCgrJsonCfg.GeneralJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, gCfg) {
+		t.Error("Received: ", gCfg)
+	}
+}
+
+func TestDfListenJsonCfg(t *testing.T) {
+	eCfg := &ListenJsonCfg{
+		Rpc_json: utils.StringPointer("127.0.0.1:2012"),
+		Rpc_gob:  utils.StringPointer("127.0.0.1:2013"),
+		Http:     utils.StringPointer("127.0.0.1:2080")}
+	if cfg, err := dfCgrJsonCfg.ListenJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfDbJsonCfg(t *testing.T) {
+	eCfg := &DbJsonCfg{
+		Db_type:   utils.StringPointer("redis"),
+		Db_host:   utils.StringPointer("127.0.0.1"),
+		Db_port:   utils.IntPointer(6379),
+		Db_name:   utils.StringPointer("10"),
+		Db_user:   utils.StringPointer(""),
+		Db_passwd: utils.StringPointer(""),
+	}
+	if cfg, err := dfCgrJsonCfg.DbJsonCfg(TPDB_JSN); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+	eCfg = &DbJsonCfg{
+		Db_type:   utils.StringPointer("redis"),
+		Db_host:   utils.StringPointer("127.0.0.1"),
+		Db_port:   utils.IntPointer(6379),
+		Db_name:   utils.StringPointer("11"),
+		Db_user:   utils.StringPointer(""),
+		Db_passwd: utils.StringPointer(""),
+	}
+	if cfg, err := dfCgrJsonCfg.DbJsonCfg(DATADB_JSN); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+	eCfg = &DbJsonCfg{
+		Db_type:        utils.StringPointer("mysql"),
+		Db_host:        utils.StringPointer("127.0.0.1"),
+		Db_port:        utils.IntPointer(3306),
+		Db_name:        utils.StringPointer("cgrates"),
+		Db_user:        utils.StringPointer("cgrates"),
+		Db_passwd:      utils.StringPointer("CGRateS.org"),
+		Max_open_conns: utils.IntPointer(100),
+		Max_idle_conns: utils.IntPointer(10),
+	}
+	if cfg, err := dfCgrJsonCfg.DbJsonCfg(STORDB_JSN); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfBalancerJsonCfg(t *testing.T) {
+	eCfg := &BalancerJsonCfg{Enabled: utils.BoolPointer(false)}
+	if cfg, err := dfCgrJsonCfg.BalancerJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfRaterJsonCfg(t *testing.T) {
+	eCfg := &RaterJsonCfg{Enabled: utils.BoolPointer(false), Balancer: utils.StringPointer(""), Cdrstats: utils.StringPointer(""),
+		Historys: utils.StringPointer(""), Pubsubs: utils.StringPointer(""), Users: utils.StringPointer("")}
+	if cfg, err := dfCgrJsonCfg.RaterJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfSchedulerJsonCfg(t *testing.T) {
+	eCfg := &SchedulerJsonCfg{Enabled: utils.BoolPointer(false)}
+	if cfg, err := dfCgrJsonCfg.SchedulerJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfCdrsJsonCfg(t *testing.T) {
+	eCfg := &CdrsJsonCfg{
+		Enabled:         utils.BoolPointer(false),
+		Extra_fields:    utils.StringSlicePointer([]string{}),
+		Store_cdrs:      utils.BoolPointer(true),
+		Rater:           utils.StringPointer("internal"),
+		Cdrstats:        utils.StringPointer(""),
+		Reconnects:      utils.IntPointer(5),
+		Cdr_replication: &[]*CdrReplicationJsonCfg{},
+	}
+	if cfg, err := dfCgrJsonCfg.CdrsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", *cfg)
+	}
+}
+
+func TestDfCdrStatsJsonCfg(t *testing.T) {
+	eCfg := &CdrStatsJsonCfg{
+		Enabled:       utils.BoolPointer(false),
+		Save_Interval: utils.StringPointer("1m"),
+	}
+	if cfg, err := dfCgrJsonCfg.CdrStatsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", *cfg)
+	}
+}
+
+func TestDfCdreJsonCfgs(t *testing.T) {
+	eFields := []*CdrFieldJsonCfg{}
+	eContentFlds := []*CdrFieldJsonCfg{
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("CgrId"),
+			Cdr_field_id: utils.StringPointer("cgrid"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("cgrid")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("RunId"),
+			Cdr_field_id: utils.StringPointer("mediation_runid"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("mediation_runid")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Tor"),
+			Cdr_field_id: utils.StringPointer("tor"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("tor")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("AccId"),
+			Cdr_field_id: utils.StringPointer("accid"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("accid")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("ReqType"),
+			Cdr_field_id: utils.StringPointer("reqtype"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("reqtype")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Direction"),
+			Cdr_field_id: utils.StringPointer("direction"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("direction")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Tenant"),
+			Cdr_field_id: utils.StringPointer("tenant"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("tenant")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Category"),
+			Cdr_field_id: utils.StringPointer("category"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("category")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Account"),
+			Cdr_field_id: utils.StringPointer("account"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("account")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Subject"),
+			Cdr_field_id: utils.StringPointer("subject"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("subject")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Destination"),
+			Cdr_field_id: utils.StringPointer("destination"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("destination")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("SetupTime"),
+			Cdr_field_id: utils.StringPointer("setup_time"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("setup_time"),
+			Layout:       utils.StringPointer("2006-01-02T15:04:05Z07:00")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("AnswerTime"),
+			Cdr_field_id: utils.StringPointer("answer_time"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("answer_time"),
+			Layout:       utils.StringPointer("2006-01-02T15:04:05Z07:00")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Usage"),
+			Cdr_field_id: utils.StringPointer("usage"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("usage")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Cost"),
+			Cdr_field_id: utils.StringPointer("cost"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("cost")},
+	}
+	eCfg := map[string]*CdreJsonCfg{
+		utils.META_DEFAULT: &CdreJsonCfg{
+			Cdr_format:                    utils.StringPointer("csv"),
+			Field_separator:               utils.StringPointer(","),
+			Data_usage_multiply_factor:    utils.Float64Pointer(1.0),
+			Sms_usage_multiply_factor:     utils.Float64Pointer(1.0),
+			Generic_usage_multiply_factor: utils.Float64Pointer(1.0),
+			Cost_multiply_factor:          utils.Float64Pointer(1.0),
+			Cost_rounding_decimals:        utils.IntPointer(-1),
+			Cost_shift_digits:             utils.IntPointer(0),
+			Mask_destination_id:           utils.StringPointer("MASKED_DESTINATIONS"),
+			Mask_length:                   utils.IntPointer(0),
+			Export_dir:                    utils.StringPointer("/var/log/cgrates/cdre"),
+			Header_fields:                 &eFields,
+			Content_fields:                &eContentFlds,
+			Trailer_fields:                &eFields,
+		},
+	}
+	if cfg, err := dfCgrJsonCfg.CdreJsonCfgs(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfCdrcJsonCfg(t *testing.T) {
+	eFields := []*CdrFieldJsonCfg{}
+	cdrFields := []*CdrFieldJsonCfg{
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("tor"), Cdr_field_id: utils.StringPointer("tor"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("2"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("accid"), Cdr_field_id: utils.StringPointer("accid"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("3"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("reqtype"), Cdr_field_id: utils.StringPointer("reqtype"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("4"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("direction"), Cdr_field_id: utils.StringPointer("direction"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("5"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("tenant"), Cdr_field_id: utils.StringPointer("tenant"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("6"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("category"), Cdr_field_id: utils.StringPointer("category"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("7"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("account"), Cdr_field_id: utils.StringPointer("account"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("8"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("subject"), Cdr_field_id: utils.StringPointer("subject"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("9"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("destination"), Cdr_field_id: utils.StringPointer("destination"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("10"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("setup_time"), Cdr_field_id: utils.StringPointer("setup_time"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("11"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("answer_time"), Cdr_field_id: utils.StringPointer("answer_time"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("12"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("usage"), Cdr_field_id: utils.StringPointer("usage"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("13"), Mandatory: utils.BoolPointer(true)},
+	}
+	eCfg := map[string]*CdrcJsonCfg{
+		"*default": &CdrcJsonCfg{
+			Enabled:                    utils.BoolPointer(false),
+			Dry_run:                    utils.BoolPointer(false),
+			Cdrs:                       utils.StringPointer("internal"),
+			Cdr_format:                 utils.StringPointer("csv"),
+			Field_separator:            utils.StringPointer(","),
+			Run_delay:                  utils.IntPointer(0),
+			Max_open_files:             utils.IntPointer(1024),
+			Data_usage_multiply_factor: utils.Float64Pointer(1024.0),
+			Cdr_in_dir:                 utils.StringPointer("/var/log/cgrates/cdrc/in"),
+			Cdr_out_dir:                utils.StringPointer("/var/log/cgrates/cdrc/out"),
+			Failed_calls_prefix:        utils.StringPointer("missed_calls"),
+			Cdr_source_id:              utils.StringPointer("freeswitch_csv"),
+			Cdr_filter:                 utils.StringPointer(""),
+			Partial_record_cache:       utils.StringPointer("10s"),
+			Header_fields:              &eFields,
+			Content_fields:             &cdrFields,
+			Trailer_fields:             &eFields,
+		},
+	}
+	if cfg, err := dfCgrJsonCfg.CdrcJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg["*default"])
+	}
+}
+
+func TestSmFsJsonCfg(t *testing.T) {
+	eCfg := &SmFsJsonCfg{
+		Enabled:                utils.BoolPointer(false),
+		Rater:                  utils.StringPointer("internal"),
+		Cdrs:                   utils.StringPointer("internal"),
+		Reconnects:             utils.IntPointer(5),
+		Create_cdr:             utils.BoolPointer(false),
+		Cdr_extra_fields:       utils.StringSlicePointer([]string{}),
+		Debit_interval:         utils.StringPointer("10s"),
+		Min_call_duration:      utils.StringPointer("0s"),
+		Max_call_duration:      utils.StringPointer("3h"),
+		Min_dur_low_balance:    utils.StringPointer("5s"),
+		Low_balance_ann_file:   utils.StringPointer(""),
+		Empty_balance_context:  utils.StringPointer(""),
+		Empty_balance_ann_file: utils.StringPointer(""),
+		Subscribe_park:         utils.BoolPointer(true),
+		Channel_sync_interval:  utils.StringPointer("5m"),
+		Connections: &[]*FsConnJsonCfg{
+			&FsConnJsonCfg{
+				Server:     utils.StringPointer("127.0.0.1:8021"),
+				Password:   utils.StringPointer("ClueCon"),
+				Reconnects: utils.IntPointer(5),
+			}},
+	}
+	if cfg, err := dfCgrJsonCfg.SmFsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestSmKamJsonCfg(t *testing.T) {
+	eCfg := &SmKamJsonCfg{
+		Enabled:           utils.BoolPointer(false),
+		Rater:             utils.StringPointer("internal"),
+		Cdrs:              utils.StringPointer("internal"),
+		Reconnects:        utils.IntPointer(5),
+		Create_cdr:        utils.BoolPointer(false),
+		Debit_interval:    utils.StringPointer("10s"),
+		Min_call_duration: utils.StringPointer("0s"),
+		Max_call_duration: utils.StringPointer("3h"),
+		Connections: &[]*KamConnJsonCfg{
+			&KamConnJsonCfg{
+				Evapi_addr: utils.StringPointer("127.0.0.1:8448"),
+				Reconnects: utils.IntPointer(5),
+			},
+		},
+	}
+	if cfg, err := dfCgrJsonCfg.SmKamJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestSmOsipsJsonCfg(t *testing.T) {
+	eCfg := &SmOsipsJsonCfg{
+		Enabled:                   utils.BoolPointer(false),
+		Listen_udp:                utils.StringPointer("127.0.0.1:2020"),
+		Rater:                     utils.StringPointer("internal"),
+		Cdrs:                      utils.StringPointer("internal"),
+		Reconnects:                utils.IntPointer(5),
+		Create_cdr:                utils.BoolPointer(false),
+		Debit_interval:            utils.StringPointer("10s"),
+		Min_call_duration:         utils.StringPointer("0s"),
+		Max_call_duration:         utils.StringPointer("3h"),
+		Events_subscribe_interval: utils.StringPointer("60s"),
+		Mi_addr:                   utils.StringPointer("127.0.0.1:8020"),
+	}
+	if cfg, err := dfCgrJsonCfg.SmOsipsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfHistServJsonCfg(t *testing.T) {
+	eCfg := &HistServJsonCfg{
+		Enabled:       utils.BoolPointer(false),
+		History_dir:   utils.StringPointer("/var/log/cgrates/history"),
+		Save_interval: utils.StringPointer("1s"),
+	}
+	if cfg, err := dfCgrJsonCfg.HistServJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfPubSubServJsonCfg(t *testing.T) {
+	eCfg := &PubSubServJsonCfg{
+		Enabled: utils.BoolPointer(false),
+	}
+	if cfg, err := dfCgrJsonCfg.PubSubServJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfUserServJsonCfg(t *testing.T) {
+	eCfg := &UserServJsonCfg{
+		Enabled: utils.BoolPointer(false),
+		Indexes: utils.StringSlicePointer([]string{}),
+	}
+	if cfg, err := dfCgrJsonCfg.UserServJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfMailerJsonCfg(t *testing.T) {
+
+	eCfg := &MailerJsonCfg{
+		Server:       utils.StringPointer("localhost"),
+		Auth_user:    utils.StringPointer("cgrates"),
+		Auth_passwd:  utils.StringPointer("CGRateS.org"),
+		From_address: utils.StringPointer("cgr-mailer@localhost.localdomain"),
+	}
+	if cfg, err := dfCgrJsonCfg.MailerJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestNewCgrJsonCfgFromFile(t *testing.T) {
+	cgrJsonCfg, err := NewCgrJsonCfgFromFile("cfg_data.json")
+	if err != nil {
+		t.Error(err)
+	}
+	eCfg := &GeneralJsonCfg{Default_reqtype: utils.StringPointer(utils.META_PSEUDOPREPAID)}
+	if gCfg, err := cgrJsonCfg.GeneralJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, gCfg) {
+		t.Error("Received: ", gCfg)
+	}
+	cdrFields := []*CdrFieldJsonCfg{
+		&CdrFieldJsonCfg{Cdr_field_id: utils.StringPointer("tor"), Value: utils.StringPointer("~7:s/^(voice|data|sms|generic)$/*$1/")},
+		&CdrFieldJsonCfg{Cdr_field_id: utils.StringPointer("answer_time"), Value: utils.StringPointer("1")},
+		&CdrFieldJsonCfg{Cdr_field_id: utils.StringPointer("usage"), Value: utils.StringPointer(`~9:s/^(\d+)$/${1}s/`)},
+	}
+	eCfgCdrc := map[string]*CdrcJsonCfg{
+		"CDRC-CSV1": &CdrcJsonCfg{
+			Enabled:       utils.BoolPointer(true),
+			Cdr_in_dir:    utils.StringPointer("/tmp/cgrates/cdrc1/in"),
+			Cdr_out_dir:   utils.StringPointer("/tmp/cgrates/cdrc1/out"),
+			Cdr_source_id: utils.StringPointer("csv1"),
+		},
+		"CDRC-CSV2": &CdrcJsonCfg{
+			Enabled:                    utils.BoolPointer(true),
+			Data_usage_multiply_factor: utils.Float64Pointer(0.000976563),
+			Run_delay:                  utils.IntPointer(1),
+			Cdr_in_dir:                 utils.StringPointer("/tmp/cgrates/cdrc2/in"),
+			Cdr_out_dir:                utils.StringPointer("/tmp/cgrates/cdrc2/out"),
+			Cdr_source_id:              utils.StringPointer("csv2"),
+			Content_fields:             &cdrFields,
+		},
+	}
+	if cfg, err := cgrJsonCfg.CdrcJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfgCdrc, cfg) {
+		t.Error("Received: ", cfg["CDRC-CSV2"])
+	}
+	eCfgSmFs := &SmFsJsonCfg{
+		Enabled: utils.BoolPointer(true),
+		Connections: &[]*FsConnJsonCfg{
+			&FsConnJsonCfg{
+				Server:     utils.StringPointer("1.2.3.4:8021"),
+				Password:   utils.StringPointer("ClueCon"),
+				Reconnects: utils.IntPointer(5),
+			},
+			&FsConnJsonCfg{
+				Server:     utils.StringPointer("2.3.4.5:8021"),
+				Password:   utils.StringPointer("ClueCon"),
+				Reconnects: utils.IntPointer(5),
+			},
+		},
+	}
+	if smFsCfg, err := cgrJsonCfg.SmFsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfgSmFs, smFsCfg) {
+		t.Error("Received: ", smFsCfg)
+	}
+}

config/config_local_test.go

@@ -1,45 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package config
-
-import (
-	"flag"
-	"path"
-	"testing"
-)
-
-var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args
-var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here")
-
-func TestLoadXmlCfg(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	cfgPath := path.Join(*dataDir, "conf", "samples", "config_local_test.cfg")
-	cfg, err := NewCGRConfigFromFile(&cfgPath)
-	if err != nil {
-		t.Error(err)
-	}
-	if cfg.XmlCfgDocument == nil {
-		t.Error("Did not load the XML Config Document")
-	}
-	if cdreFWCfg := cfg.XmlCfgDocument.GetCdreCfgs("CDREFW-A"); cdreFWCfg == nil {
-		t.Error("Could not retrieve CDRExporter FixedWidth config instance")
-	}
-}

config/config_test.go

@@ -1,6 +1,6 @@
 /*
 Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -19,17 +19,14 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package config
 
 import (
-	"fmt"
 	"reflect"
-	"regexp"
 	"testing"
-	"time"
-
-	"github.com/cgrates/cgrates/utils"
 )
 
+var cfg *CGRConfig
+
 func TestConfigSharing(t *testing.T) {
-	cfg, _ := NewDefaultCGRConfig()
+	cfg, _ = NewDefaultCGRConfig()
 	SetCgrConfig(cfg)
 	cfgReturn := CgrConfig()
 	if !reflect.DeepEqual(cfgReturn, cfg) {
@@ -37,358 +34,27 @@ func TestConfigSharing(t *testing.T) {
 	}
 }
 
-// Make sure defaults did not change by mistake
-func TestDefaults(t *testing.T) {
-	cfg := &CGRConfig{}
-	errSet := cfg.setDefaults()
-	if errSet != nil {
-		t.Log(fmt.Sprintf("Coud not set defaults: %s!", errSet.Error()))
-		t.FailNow()
-	}
-	eCfg := &CGRConfig{}
-	eCfg.RatingDBType = REDIS
-	eCfg.RatingDBHost = "127.0.0.1"
-	eCfg.RatingDBPort = "6379"
-	eCfg.RatingDBName = "10"
-	eCfg.RatingDBUser = ""
-	eCfg.RatingDBPass = ""
-	eCfg.AccountDBType = REDIS
-	eCfg.AccountDBHost = "127.0.0.1"
-	eCfg.AccountDBPort = "6379"
-	eCfg.AccountDBName = "11"
-	eCfg.AccountDBUser = ""
-	eCfg.AccountDBPass = ""
-	eCfg.StorDBType = utils.MYSQL
-	eCfg.StorDBHost = "localhost"
-	eCfg.StorDBPort = "3306"
-	eCfg.StorDBName = "cgrates"
-	eCfg.StorDBUser = "cgrates"
-	eCfg.StorDBPass = "CGRateS.org"
-	eCfg.DBDataEncoding = utils.MSGPACK
-	eCfg.RPCJSONListen = "127.0.0.1:2012"
-	eCfg.RPCGOBListen = "127.0.0.1:2013"
-	eCfg.HTTPListen = "127.0.0.1:2080"
-	eCfg.DefaultReqType = utils.RATED
-	eCfg.DefaultCategory = "call"
-	eCfg.DefaultTenant = "cgrates.org"
-	eCfg.DefaultSubject = "cgrates"
-	eCfg.RoundingDecimals = 10
-	eCfg.HttpSkipTlsVerify = false
-	eCfg.XmlCfgDocument = nil
-	eCfg.RaterEnabled = false
-	eCfg.RaterBalancer = ""
-	eCfg.BalancerEnabled = false
-	eCfg.SchedulerEnabled = false
-	eCfg.CdreDefaultInstance, _ = NewDefaultCdreConfig()
-	eCfg.CDRSEnabled = false
-	eCfg.CDRSExtraFields = []*utils.RSRField{}
-	eCfg.CDRSMediator = ""
-	eCfg.CDRSStats = ""
-	eCfg.CDRSStoreDisable = false
-	eCfg.CDRStatsEnabled = false
-	eCfg.CDRStatConfig = &CdrStatsConfig{Id: utils.META_DEFAULT, QueueLength: 50, TimeWindow: time.Duration(1) * time.Hour, Metrics: []string{"ASR", "ACD", "ACC"}}
-	eCfg.CdrcEnabled = false
-	eCfg.CdrcCdrs = utils.INTERNAL
-	eCfg.CdrcRunDelay = time.Duration(0)
-	eCfg.CdrcCdrType = "csv"
-	eCfg.CdrcCsvSep = string(utils.CSV_SEP)
-	eCfg.CdrcCdrInDir = "/var/log/cgrates/cdrc/in"
-	eCfg.CdrcCdrOutDir = "/var/log/cgrates/cdrc/out"
-	eCfg.CdrcSourceId = "csv"
-	eCfg.CdrcCdrFields = map[string][]*utils.RSRField{
-		utils.TOR:         []*utils.RSRField{&utils.RSRField{Id: "2"}},
-		utils.ACCID:       []*utils.RSRField{&utils.RSRField{Id: "3"}},
-		utils.REQTYPE:     []*utils.RSRField{&utils.RSRField{Id: "4"}},
-		utils.DIRECTION:   []*utils.RSRField{&utils.RSRField{Id: "5"}},
-		utils.TENANT:      []*utils.RSRField{&utils.RSRField{Id: "6"}},
-		utils.CATEGORY:    []*utils.RSRField{&utils.RSRField{Id: "7"}},
-		utils.ACCOUNT:     []*utils.RSRField{&utils.RSRField{Id: "8"}},
-		utils.SUBJECT:     []*utils.RSRField{&utils.RSRField{Id: "9"}},
-		utils.DESTINATION: []*utils.RSRField{&utils.RSRField{Id: "10"}},
-		utils.SETUP_TIME:  []*utils.RSRField{&utils.RSRField{Id: "11"}},
-		utils.ANSWER_TIME: []*utils.RSRField{&utils.RSRField{Id: "12"}},
-		utils.USAGE:       []*utils.RSRField{&utils.RSRField{Id: "13"}},
-	}
-	eCfg.MediatorEnabled = false
-	eCfg.MediatorRater = utils.INTERNAL
-	eCfg.MediatorReconnects = 3
-	eCfg.MediatorStats = utils.INTERNAL
-	eCfg.MediatorStoreDisable = false
-	eCfg.SMEnabled = false
-	eCfg.SMSwitchType = FS
-	eCfg.SMRater = utils.INTERNAL
-	eCfg.SMCdrS = ""
-	eCfg.SMReconnects = 3
-	eCfg.SMDebitInterval = 10
-	eCfg.SMMinCallDuration = time.Duration(0)
-	eCfg.SMMaxCallDuration = time.Duration(3) * time.Hour
-	eCfg.FreeswitchServer = "127.0.0.1:8021"
-	eCfg.FreeswitchPass = "ClueCon"
-	eCfg.FreeswitchReconnects = 5
-	eCfg.FSMinDurLowBalance = time.Duration(5) * time.Second
-	eCfg.FSLowBalanceAnnFile = ""
-	eCfg.FSEmptyBalanceContext = ""
-	eCfg.FSEmptyBalanceAnnFile = ""
-	eCfg.FSCdrExtraFields = []*utils.RSRField{}
-	eCfg.OsipsListenUdp = "127.0.0.1:2020"
-	eCfg.OsipsMiAddr = "127.0.0.1:8020"
-	eCfg.OsipsEvSubscInterval = time.Duration(60) * time.Second
-	eCfg.OsipsReconnects = 3
-	eCfg.DerivedChargers = make(utils.DerivedChargers, 0)
-	eCfg.CombinedDerivedChargers = true
-	eCfg.HistoryAgentEnabled = false
-	eCfg.HistoryServer = utils.INTERNAL
-	eCfg.HistoryServerEnabled = false
-	eCfg.HistoryDir = "/var/log/cgrates/history"
-	eCfg.HistorySaveInterval = time.Duration(1) * time.Second
-	eCfg.MailerServer = "localhost:25"
-	eCfg.MailerAuthUser = "cgrates"
-	eCfg.MailerAuthPass = "CGRateS.org"
-	eCfg.MailerFromAddr = "cgr-mailer@localhost.localdomain"
-	eCfg.DataFolderPath = "/usr/share/cgrates/"
-	if !reflect.DeepEqual(cfg, eCfg) {
-		t.Log(eCfg)
-		t.Log(cfg)
-		t.Error("Defaults different than expected!")
-	}
+func TestLoadCgrCfgWithDefaults(t *testing.T) {
+	JSN_CFG := `
+{
+"sm_freeswitch": {
+	"enabled": true,				// starts SessionManager service: <true|false>
+	"connections":[					// instantiate connections to multiple FreeSWITCH servers
+		{"server": "1.2.3.4:8021", "password": "ClueCon", "reconnects": 3},
+		{"server": "1.2.3.5:8021", "password": "ClueCon", "reconnects": 5}
+	],
+},
 
-}
-
-func TestSanityCheck(t *testing.T) {
-	cfg := &CGRConfig{}
-	errSet := cfg.setDefaults()
-	if errSet != nil {
-		t.Error("Coud not set defaults: ", errSet.Error())
+}`
+	eCgrCfg, _ := NewDefaultCGRConfig()
+	eCgrCfg.SmFsConfig.Enabled = true
+	eCgrCfg.SmFsConfig.Connections = []*FsConnConfig{
+		&FsConnConfig{Server: "1.2.3.4:8021", Password: "ClueCon", Reconnects: 3},
+		&FsConnConfig{Server: "1.2.3.5:8021", Password: "ClueCon", Reconnects: 5},
 	}
-	if err := cfg.checkConfigSanity(); err != nil {
-		t.Error("Invalid defaults: ", err)
-	}
-	cfg = &CGRConfig{}
-	cfg.CdrcEnabled = true
-	if err := cfg.checkConfigSanity(); err == nil {
-		t.Error("Failed to detect missing CDR fields definition")
-	}
-	cfg.CdrcCdrType = utils.CSV
-	cfg.CdrcCdrFields = map[string][]*utils.RSRField{utils.ACCID: []*utils.RSRField{&utils.RSRField{Id: "test"}}}
-	if err := cfg.checkConfigSanity(); err == nil {
-		t.Error("Failed to detect improper use of CDR field names")
-	}
-	cfg.CdrcCdrFields = map[string][]*utils.RSRField{"extra1": []*utils.RSRField{&utils.RSRField{Id: "test"}}}
-	if err := cfg.checkConfigSanity(); err == nil {
-		t.Error("Failed to detect improper use of CDR field names")
-	}
-}
-
-// Load config from file and make sure we have all set
-func TestConfigFromFile(t *testing.T) {
-	cfgPth := "test_data.txt"
-	cfg, err := NewCGRConfigFromFile(&cfgPth)
-	if err != nil {
-		t.Log(fmt.Sprintf("Could not parse config: %s!", err))
-		t.FailNow()
-	}
-	eCfg := &CGRConfig{} // Instance we expect to get out after reading config file
-	eCfg.setDefaults()
-	eCfg.RatingDBType = "test"
-	eCfg.RatingDBHost = "test"
-	eCfg.RatingDBPort = "test"
-	eCfg.RatingDBName = "test"
-	eCfg.RatingDBUser = "test"
-	eCfg.RatingDBPass = "test"
-	eCfg.AccountDBType = "test"
-	eCfg.AccountDBHost = "test"
-	eCfg.AccountDBPort = "test"
-	eCfg.AccountDBName = "test"
-	eCfg.AccountDBUser = "test"
-	eCfg.AccountDBPass = "test"
-	eCfg.StorDBType = "test"
-	eCfg.StorDBHost = "test"
-	eCfg.StorDBPort = "test"
-	eCfg.StorDBName = "test"
-	eCfg.StorDBUser = "test"
-	eCfg.StorDBPass = "test"
-	eCfg.DBDataEncoding = "test"
-	eCfg.RPCJSONListen = "test"
-	eCfg.RPCGOBListen = "test"
-	eCfg.HTTPListen = "test"
-	eCfg.DefaultReqType = "test"
-	eCfg.DefaultCategory = "test"
-	eCfg.DefaultTenant = "test"
-	eCfg.DefaultSubject = "test"
-	eCfg.RoundingDecimals = 99
-	eCfg.HttpSkipTlsVerify = true
-	eCfg.RaterEnabled = true
-	eCfg.RaterBalancer = "test"
-	eCfg.BalancerEnabled = true
-	eCfg.SchedulerEnabled = true
-	eCfg.CDRSEnabled = true
-	eCfg.CDRSExtraFields = []*utils.RSRField{&utils.RSRField{Id: "test"}}
-	eCfg.CDRSMediator = "test"
-	eCfg.CDRSStats = "test"
-	eCfg.CDRSStoreDisable = true
-	eCfg.CDRStatsEnabled = true
-	eCfg.CDRStatConfig = &CdrStatsConfig{Id: utils.META_DEFAULT, QueueLength: 99, TimeWindow: time.Duration(99) * time.Second,
-		Metrics: []string{"test"}, TORs: []string{"test"}, CdrHosts: []string{"test"}, CdrSources: []string{"test"}, ReqTypes: []string{"test"}, Directions: []string{"test"},
-		Tenants: []string{"test"}, Categories: []string{"test"}, Accounts: []string{"test"}, Subjects: []string{"test"}, DestinationPrefixes: []string{"test"},
-		UsageInterval:   []time.Duration{time.Duration(99) * time.Second},
-		MediationRunIds: []string{"test"}, RatedAccounts: []string{"test"}, RatedSubjects: []string{"test"}, CostInterval: []float64{99.0}}
-	eCfg.CDRSStats = "test"
-	eCfg.CdreDefaultInstance = &CdreConfig{
-		CdrFormat:               "test",
-		FieldSeparator:          utils.CSV_SEP,
-		DataUsageMultiplyFactor: 99.0,
-		CostMultiplyFactor:      99.0,
-		CostRoundingDecimals:    99,
-		CostShiftDigits:         99,
-		MaskDestId:              "test",
-		MaskLength:              99,
-		ExportDir:               "test"}
-	eCfg.CdreDefaultInstance.ContentFields, _ = NewCdreCdrFieldsFromIds(false, "test")
-	eCfg.CdrcEnabled = true
-	eCfg.CdrcCdrs = "test"
-	eCfg.CdrcRunDelay = time.Duration(99) * time.Second
-	eCfg.CdrcCdrType = "test"
-	eCfg.CdrcCsvSep = ";"
-	eCfg.CdrcCdrInDir = "test"
-	eCfg.CdrcCdrOutDir = "test"
-	eCfg.CdrcSourceId = "test"
-	eCfg.CdrcCdrFields = map[string][]*utils.RSRField{
-		utils.TOR:         []*utils.RSRField{&utils.RSRField{Id: "test"}},
-		utils.ACCID:       []*utils.RSRField{&utils.RSRField{Id: "test"}},
-		utils.REQTYPE:     []*utils.RSRField{&utils.RSRField{Id: "test"}},
-		utils.DIRECTION:   []*utils.RSRField{&utils.RSRField{Id: "test"}},
-		utils.TENANT:      []*utils.RSRField{&utils.RSRField{Id: "test"}},
-		utils.CATEGORY:    []*utils.RSRField{&utils.RSRField{Id: "test"}},
-		utils.ACCOUNT:     []*utils.RSRField{&utils.RSRField{Id: "test"}},
-		utils.SUBJECT:     []*utils.RSRField{&utils.RSRField{Id: "test"}},
-		utils.DESTINATION: []*utils.RSRField{&utils.RSRField{Id: "test"}},
-		utils.SETUP_TIME:  []*utils.RSRField{&utils.RSRField{Id: "test"}},
-		utils.ANSWER_TIME: []*utils.RSRField{&utils.RSRField{Id: "test"}},
-		utils.USAGE:       []*utils.RSRField{&utils.RSRField{Id: "test"}},
-		"test":            []*utils.RSRField{&utils.RSRField{Id: "test"}},
-	}
-	eCfg.MediatorEnabled = true
-	eCfg.MediatorRater = "test"
-	eCfg.MediatorReconnects = 99
-	eCfg.MediatorStats = "test"
-	eCfg.MediatorStoreDisable = true
-	eCfg.SMEnabled = true
-	eCfg.SMSwitchType = "test"
-	eCfg.SMRater = "test"
-	eCfg.SMCdrS = "test"
-	eCfg.SMReconnects = 99
-	eCfg.SMDebitInterval = 99
-	eCfg.SMMinCallDuration = time.Duration(98) * time.Second
-	eCfg.SMMaxCallDuration = time.Duration(99) * time.Second
-	eCfg.FreeswitchServer = "test"
-	eCfg.FreeswitchPass = "test"
-	eCfg.FreeswitchReconnects = 99
-	eCfg.FSMinDurLowBalance = time.Duration(99) * time.Second
-	eCfg.FSLowBalanceAnnFile = "test"
-	eCfg.FSEmptyBalanceContext = "test"
-	eCfg.FSEmptyBalanceAnnFile = "test"
-	eCfg.FSCdrExtraFields = []*utils.RSRField{&utils.RSRField{Id: "test"}}
-	eCfg.OsipsListenUdp = "test"
-	eCfg.OsipsMiAddr = "test"
-	eCfg.OsipsEvSubscInterval = time.Duration(99) * time.Second
-	eCfg.OsipsReconnects = 99
-	eCfg.DerivedChargers = utils.DerivedChargers{&utils.DerivedCharger{RunId: "test", RunFilters: "", ReqTypeField: "test", DirectionField: "test", TenantField: "test",
-		CategoryField: "test", AccountField: "test", SubjectField: "test", DestinationField: "test", SetupTimeField: "test", AnswerTimeField: "test", UsageField: "test"}}
-	eCfg.CombinedDerivedChargers = true
-	eCfg.HistoryAgentEnabled = true
-	eCfg.HistoryServer = "test"
-	eCfg.HistoryServerEnabled = true
-	eCfg.HistoryDir = "test"
-	eCfg.HistorySaveInterval = time.Duration(99) * time.Second
-	eCfg.MailerServer = "test"
-	eCfg.MailerAuthUser = "test"
-	eCfg.MailerAuthPass = "test"
-	eCfg.MailerFromAddr = "test"
-	eCfg.DataFolderPath = "/usr/share/cgrates/"
-	if !reflect.DeepEqual(cfg, eCfg) {
-		t.Log(eCfg)
-		t.Log(cfg)
-		t.Error("Loading of configuration from file failed!")
-	}
-}
-
-func TestCdrsExtraFields(t *testing.T) {
-	eFieldsCfg := []byte(`[cdrs]
-extra_fields = extr1,extr2
-`)
-	if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil {
-		t.Error("Could not parse the config", err.Error())
-	} else if !reflect.DeepEqual(cfg.CDRSExtraFields, []*utils.RSRField{&utils.RSRField{Id: "extr1"}, &utils.RSRField{Id: "extr2"}}) {
-		t.Errorf("Unexpected value for CdrsExtraFields: %v", cfg.CDRSExtraFields)
-	}
-	eFieldsCfg = []byte(`[cdrs]
-extra_fields = ~effective_caller_id_number:s/(\d+)/+$1/
-`)
-	if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil {
-		t.Error("Could not parse the config", err.Error())
-	} else if !reflect.DeepEqual(cfg.CDRSExtraFields, []*utils.RSRField{&utils.RSRField{Id: "effective_caller_id_number",
-		RSRules: []*utils.ReSearchReplace{&utils.ReSearchReplace{SearchRegexp: regexp.MustCompile(`(\d+)`), ReplaceTemplate: "+$1"}}}}) {
-		t.Errorf("Unexpected value for config CdrsExtraFields: %v", cfg.CDRSExtraFields)
-	}
-	eFieldsCfg = []byte(`[cdrs]
-extra_fields = extr1,~extr2:s/x.+/
-`)
-	if _, err := NewCGRConfigFromBytes(eFieldsCfg); err == nil {
-		t.Error("Failed to detect failed RSRParsing")
-	}
-
-}
-
-func TestCdreExtraFields(t *testing.T) {
-	eFieldsCfg := []byte(`[cdre]
-cdr_format = csv
-export_template = cgrid,mediation_runid,accid
-`)
-	expectedFlds := []*CdreCdrField{
-		&CdreCdrField{Name: "cgrid", Type: utils.CDRFIELD, Value: "cgrid", valueAsRsrField: &utils.RSRField{Id: "cgrid"}, Mandatory: true},
-		&CdreCdrField{Name: "mediation_runid", Type: utils.CDRFIELD, Value: "mediation_runid", valueAsRsrField: &utils.RSRField{Id: "mediation_runid"},
-			Mandatory: true},
-		&CdreCdrField{Name: "accid", Type: utils.CDRFIELD, Value: "accid", valueAsRsrField: &utils.RSRField{Id: "accid"},
-			Mandatory: true},
-	}
-	expCdreCfg := &CdreConfig{CdrFormat: utils.CSV, FieldSeparator: utils.CSV_SEP, CostRoundingDecimals: -1, ExportDir: "/var/log/cgrates/cdre", ContentFields: expectedFlds}
-	if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil {
-		t.Error("Could not parse the config", err.Error())
-	} else if !reflect.DeepEqual(cfg.CdreDefaultInstance, expCdreCfg) {
-		t.Errorf("Expecting: %v, received: %v", expCdreCfg, cfg.CdreDefaultInstance)
-	}
-	eFieldsCfg = []byte(`[cdre]
-cdr_format = csv
-export_template = cgrid,~effective_caller_id_number:s/(\d+)/+$1/
-`)
-	rsrField, _ := utils.NewRSRField(`~effective_caller_id_number:s/(\d+)/+$1/`)
-	expectedFlds = []*CdreCdrField{
-		&CdreCdrField{Name: "cgrid", Type: utils.CDRFIELD, Value: "cgrid", valueAsRsrField: &utils.RSRField{Id: "cgrid"}, Mandatory: true},
-		&CdreCdrField{Name: `~effective_caller_id_number:s/(\d+)/+$1/`, Type: utils.CDRFIELD, Value: `~effective_caller_id_number:s/(\d+)/+$1/`,
-			valueAsRsrField: rsrField, Mandatory: false}}
-	expCdreCfg.ContentFields = expectedFlds
-	if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil {
-		t.Error("Could not parse the config", err.Error())
-	} else if !reflect.DeepEqual(cfg.CdreDefaultInstance, expCdreCfg) {
-		t.Errorf("Expecting: %v, received: %v", expCdreCfg, cfg.CdreDefaultInstance)
-	}
-	eFieldsCfg = []byte(`[cdre]
-cdr_format = csv
-export_template = cgrid,~accid:s/(\d)/$1,runid
-`)
-	if _, err := NewCGRConfigFromBytes(eFieldsCfg); err == nil {
-		t.Error("Failed to detect failed RSRParsing")
-	}
-}
-
-func TestCdrcCdrDefaultFields(t *testing.T) {
-	cdrcCfg := []byte(`[cdrc]
-enabled = true
-`)
-	cfgDefault, _ := NewDefaultCGRConfig()
-	if cfg, err := NewCGRConfigFromBytes(cdrcCfg); err != nil {
-		t.Error("Could not parse the config", err.Error())
-	} else if !reflect.DeepEqual(cfg.CdrcCdrFields, cfgDefault.CdrcCdrFields) {
-		t.Errorf("Unexpected value for CdrcCdrFields: %v", cfg.CdrcCdrFields)
+	if cgrCfg, err := NewCGRConfigFromJsonStringWithDefaults(JSN_CFG); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCgrCfg.SmFsConfig, cgrCfg.SmFsConfig) {
+		t.Errorf("Expected: %+v, received: %+v", eCgrCfg.SmFsConfig, cgrCfg.SmFsConfig)
 	}
 }

config/configcdrc_test.go

@@ -0,0 +1,194 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+func TestLoadCdrcConfigMultipleFiles(t *testing.T) {
+	cgrCfg, err := NewCGRConfigFromFolder(".")
+	if err != nil {
+		t.Error(err)
+	}
+	eCgrCfg, _ := NewDefaultCGRConfig()
+	eCgrCfg.CdrcProfiles = make(map[string]map[string]*CdrcConfig)
+	// Default instance first
+	eCgrCfg.CdrcProfiles["/var/log/cgrates/cdrc/in"] = map[string]*CdrcConfig{
+		"*default": &CdrcConfig{
+			Enabled:                 false,
+			Cdrs:                    "internal",
+			CdrFormat:               "csv",
+			FieldSeparator:          ',',
+			DataUsageMultiplyFactor: 1024,
+			RunDelay:                0,
+			MaxOpenFiles:            1024,
+			CdrInDir:                "/var/log/cgrates/cdrc/in",
+			CdrOutDir:               "/var/log/cgrates/cdrc/out",
+			FailedCallsPrefix:       "missed_calls",
+			CdrSourceId:             "freeswitch_csv",
+			CdrFilter:               utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP),
+			PartialRecordCache:      time.Duration(10) * time.Second,
+			HeaderFields:            make([]*CfgCdrField, 0),
+			ContentFields: []*CfgCdrField{
+				&CfgCdrField{Tag: "tor", Type: "cdrfield", CdrFieldId: "tor", Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "accid", Type: "cdrfield", CdrFieldId: "accid", Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "reqtype", Type: "cdrfield", CdrFieldId: "reqtype", Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "direction", Type: "cdrfield", CdrFieldId: "direction", Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "tenant", Type: "cdrfield", CdrFieldId: "tenant", Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "category", Type: "cdrfield", CdrFieldId: "category", Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "account", Type: "cdrfield", CdrFieldId: "account", Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "subject", Type: "cdrfield", CdrFieldId: "subject", Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: utils.ParseRSRFieldsMustCompile("10", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "setup_time", Type: "cdrfield", CdrFieldId: "setup_time", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "answer_time", Type: "cdrfield", CdrFieldId: "answer_time", Value: utils.ParseRSRFieldsMustCompile("12", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "usage", Type: "cdrfield", CdrFieldId: "usage", Value: utils.ParseRSRFieldsMustCompile("13", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+			},
+			TrailerFields: make([]*CfgCdrField, 0),
+		},
+	}
+	eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc1/in"] = map[string]*CdrcConfig{
+		"CDRC-CSV1": &CdrcConfig{
+			Enabled:                 true,
+			Cdrs:                    "internal",
+			CdrFormat:               "csv",
+			FieldSeparator:          ',',
+			DataUsageMultiplyFactor: 1024,
+			RunDelay:                0,
+			MaxOpenFiles:            1024,
+			CdrInDir:                "/tmp/cgrates/cdrc1/in",
+			CdrOutDir:               "/tmp/cgrates/cdrc1/out",
+			CdrSourceId:             "csv1",
+			CdrFilter:               utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP),
+			HeaderFields:            make([]*CfgCdrField, 0),
+			ContentFields: []*CfgCdrField{
+				&CfgCdrField{Tag: "tor", Type: "cdrfield", CdrFieldId: "tor", Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "accid", Type: "cdrfield", CdrFieldId: "accid", Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "reqtype", Type: "cdrfield", CdrFieldId: "reqtype", Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "direction", Type: "cdrfield", CdrFieldId: "direction", Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "tenant", Type: "cdrfield", CdrFieldId: "tenant", Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "category", Type: "cdrfield", CdrFieldId: "category", Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "account", Type: "cdrfield", CdrFieldId: "account", Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "subject", Type: "cdrfield", CdrFieldId: "subject", Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: utils.ParseRSRFieldsMustCompile("10", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "setup_time", Type: "cdrfield", CdrFieldId: "setup_time", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "answer_time", Type: "cdrfield", CdrFieldId: "answer_time", Value: utils.ParseRSRFieldsMustCompile("12", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "usage", Type: "cdrfield", CdrFieldId: "usage", Value: utils.ParseRSRFieldsMustCompile("13", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+			},
+			TrailerFields: make([]*CfgCdrField, 0),
+		},
+	}
+	eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc2/in"] = map[string]*CdrcConfig{
+		"CDRC-CSV2": &CdrcConfig{
+			Enabled:                 true,
+			Cdrs:                    "internal",
+			CdrFormat:               "csv",
+			FieldSeparator:          ',',
+			DataUsageMultiplyFactor: 0.000976563,
+			RunDelay:                0,
+			MaxOpenFiles:            1024,
+			CdrInDir:                "/tmp/cgrates/cdrc2/in",
+			CdrOutDir:               "/tmp/cgrates/cdrc2/out",
+			CdrSourceId:             "csv2",
+			CdrFilter:               utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP),
+			HeaderFields:            make([]*CfgCdrField, 0),
+			ContentFields: []*CfgCdrField{
+				&CfgCdrField{Tag: "", Type: "", CdrFieldId: "tor", Value: utils.ParseRSRFieldsMustCompile("~7:s/^(voice|data|sms|generic)$/*$1/", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: false},
+				&CfgCdrField{Tag: "", Type: "", CdrFieldId: "answer_time", Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: false},
+			},
+			TrailerFields: make([]*CfgCdrField, 0),
+		},
+	}
+	eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc3/in"] = map[string]*CdrcConfig{
+		"CDRC-CSV3": &CdrcConfig{
+			Enabled:                 true,
+			Cdrs:                    "internal",
+			CdrFormat:               "csv",
+			FieldSeparator:          ',',
+			DataUsageMultiplyFactor: 1024,
+			RunDelay:                0,
+			MaxOpenFiles:            1024,
+			CdrInDir:                "/tmp/cgrates/cdrc3/in",
+			CdrOutDir:               "/tmp/cgrates/cdrc3/out",
+			CdrSourceId:             "csv3",
+			CdrFilter:               utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP),
+			HeaderFields:            make([]*CfgCdrField, 0),
+			ContentFields: []*CfgCdrField{
+				&CfgCdrField{Tag: "tor", Type: "cdrfield", CdrFieldId: "tor", Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "accid", Type: "cdrfield", CdrFieldId: "accid", Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "reqtype", Type: "cdrfield", CdrFieldId: "reqtype", Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "direction", Type: "cdrfield", CdrFieldId: "direction", Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "tenant", Type: "cdrfield", CdrFieldId: "tenant", Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "category", Type: "cdrfield", CdrFieldId: "category", Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "account", Type: "cdrfield", CdrFieldId: "account", Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "subject", Type: "cdrfield", CdrFieldId: "subject", Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: utils.ParseRSRFieldsMustCompile("10", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "setup_time", Type: "cdrfield", CdrFieldId: "setup_time", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "answer_time", Type: "cdrfield", CdrFieldId: "answer_time", Value: utils.ParseRSRFieldsMustCompile("12", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "usage", Type: "cdrfield", CdrFieldId: "usage", Value: utils.ParseRSRFieldsMustCompile("13", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+			},
+			TrailerFields: make([]*CfgCdrField, 0),
+		},
+	}
+	if !reflect.DeepEqual(eCgrCfg.CdrcProfiles, cgrCfg.CdrcProfiles) {
+		t.Errorf("Expected: %+v, received: %+v", eCgrCfg.CdrcProfiles, cgrCfg.CdrcProfiles)
+	}
+}

config/helpers.go

@@ -1,155 +0,0 @@
-/*
-Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package config
-
-import (
-	"code.google.com/p/goconf/conf"
-	"errors"
-	"fmt"
-	"strings"
-
-	"github.com/cgrates/cgrates/utils"
-)
-
-// Adds support for slice values in config
-func ConfigSlice(cfgVal string) ([]string, error) {
-	cfgValStrs := strings.Split(cfgVal, utils.FIELDS_SEP) // If need arrises, we can make the separator configurable
-	for idx, elm := range cfgValStrs {
-		cfgValStrs[idx] = strings.TrimSpace(elm) // By default spaces are not removed so we do it here to avoid unpredicted results in config
-	}
-	return cfgValStrs, nil
-}
-
-// Parse the configuration file and returns utils.DerivedChargers instance if no errors
-func ParseCfgDerivedCharging(c *conf.ConfigFile) (dcs utils.DerivedChargers, err error) {
-	var runIds, runFilters, reqTypeFlds, directionFlds, tenantFlds, torFlds, acntFlds, subjFlds, dstFlds, sTimeFlds, aTimeFlds, durFlds []string
-	cfgVal, _ := c.GetString("derived_charging", "run_ids")
-	if runIds, err = ConfigSlice(cfgVal); err != nil {
-		return nil, err
-	}
-	cfgVal, _ = c.GetString("derived_charging", "run_filters")
-	if runFilters, err = ConfigSlice(cfgVal); err != nil {
-		return nil, err
-	}
-	cfgVal, _ = c.GetString("derived_charging", "reqtype_fields")
-	if reqTypeFlds, err = ConfigSlice(cfgVal); err != nil {
-		return nil, err
-	}
-	cfgVal, _ = c.GetString("derived_charging", "direction_fields")
-	if directionFlds, err = ConfigSlice(cfgVal); err != nil {
-		return nil, err
-	}
-	cfgVal, _ = c.GetString("derived_charging", "tenant_fields")
-	if tenantFlds, err = ConfigSlice(cfgVal); err != nil {
-		return nil, err
-	}
-	cfgVal, _ = c.GetString("derived_charging", "category_fields")
-	if torFlds, err = ConfigSlice(cfgVal); err != nil {
-		return nil, err
-	}
-	cfgVal, _ = c.GetString("derived_charging", "account_fields")
-	if acntFlds, err = ConfigSlice(cfgVal); err != nil {
-		return nil, err
-	}
-	cfgVal, _ = c.GetString("derived_charging", "subject_fields")
-	if subjFlds, err = ConfigSlice(cfgVal); err != nil {
-		return nil, err
-	}
-	cfgVal, _ = c.GetString("derived_charging", "destination_fields")
-	if dstFlds, err = ConfigSlice(cfgVal); err != nil {
-		return nil, err
-	}
-	cfgVal, _ = c.GetString("derived_charging", "setup_time_fields")
-	if sTimeFlds, err = ConfigSlice(cfgVal); err != nil {
-		return nil, err
-	}
-	cfgVal, _ = c.GetString("derived_charging", "answer_time_fields")
-	if aTimeFlds, err = ConfigSlice(cfgVal); err != nil {
-		return nil, err
-	}
-	cfgVal, _ = c.GetString("derived_charging", "usage_fields")
-	if durFlds, err = ConfigSlice(cfgVal); err != nil {
-		return nil, err
-	}
-	// We need all to be the same length
-	if len(runFilters) != len(runIds) ||
-		len(reqTypeFlds) != len(runIds) ||
-		len(directionFlds) != len(runIds) ||
-		len(tenantFlds) != len(runIds) ||
-		len(torFlds) != len(runIds) ||
-		len(acntFlds) != len(runIds) ||
-		len(subjFlds) != len(runIds) ||
-		len(dstFlds) != len(runIds) ||
-		len(sTimeFlds) != len(runIds) ||
-		len(aTimeFlds) != len(runIds) ||
-		len(durFlds) != len(runIds) {
-		return nil, errors.New("<ConfigSanity> Inconsistent fields length in derivated_charging section")
-	}
-	// Create the individual chargers and append them to the final instance
-	dcs = make(utils.DerivedChargers, 0)
-	if len(runIds) == 1 && len(runIds[0]) == 0 { // Avoid iterating on empty runid
-		return dcs, nil
-	}
-	for runIdx, runId := range runIds {
-		dc, err := utils.NewDerivedCharger(runId, runFilters[runIdx], reqTypeFlds[runIdx], directionFlds[runIdx], tenantFlds[runIdx], torFlds[runIdx],
-			acntFlds[runIdx], subjFlds[runIdx], dstFlds[runIdx], sTimeFlds[runIdx], aTimeFlds[runIdx], durFlds[runIdx])
-		if err != nil {
-			return nil, err
-		}
-		if dcs, err = dcs.Append(dc); err != nil {
-			return nil, err
-		}
-	}
-	return dcs, nil
-}
-
-func ParseCdrcCdrFields(torFld, accIdFld, reqtypeFld, directionFld, tenantFld, categoryFld, acntFld, subjectFld, destFld,
-	setupTimeFld, answerTimeFld, durFld, extraFlds string) (map[string][]*utils.RSRField, error) {
-	cdrcCdrFlds := make(map[string][]*utils.RSRField)
-	if len(extraFlds) != 0 {
-		if sepExtraFlds, err := ConfigSlice(extraFlds); err != nil {
-			return nil, err
-		} else {
-			for _, fldStr := range sepExtraFlds {
-				// extra fields defined as: <label_extrafield_1>:<index_extrafield_1>
-				if spltLbl := strings.Split(fldStr, utils.CONCATENATED_KEY_SEP); len(spltLbl) != 2 {
-					return nil, fmt.Errorf("Wrong format for cdrc.extra_fields: %s", fldStr)
-				} else {
-					if rsrFlds, err := utils.ParseRSRFields(spltLbl[1], utils.INFIELD_SEP); err != nil {
-						return nil, err
-					} else {
-						cdrcCdrFlds[spltLbl[0]] = rsrFlds
-					}
-				}
-			}
-		}
-	}
-	for fldTag, fldVal := range map[string]string{utils.TOR: torFld, utils.ACCID: accIdFld, utils.REQTYPE: reqtypeFld, utils.DIRECTION: directionFld, utils.TENANT: tenantFld,
-		utils.CATEGORY: categoryFld, utils.ACCOUNT: acntFld, utils.SUBJECT: subjectFld, utils.DESTINATION: destFld, utils.SETUP_TIME: setupTimeFld,
-		utils.ANSWER_TIME: answerTimeFld, utils.USAGE: durFld} {
-		if len(fldVal) != 0 {
-			if rsrFlds, err := utils.ParseRSRFields(fldVal, utils.INFIELD_SEP); err != nil {
-				return nil, err
-			} else {
-				cdrcCdrFlds[fldTag] = rsrFlds
-			}
-		}
-	}
-	return cdrcCdrFlds, nil
-}

config/helpers_test.go

@@ -1,140 +0,0 @@
-/*
-Real-time Charging System for Telecom & ISP environments
-Copyright (C) 2012-2014 ITsysCOM GmbH
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package config
-
-import (
-	"encoding/json"
-	"reflect"
-	"testing"
-
-	"github.com/cgrates/cgrates/utils"
-)
-
-func TestConfigSlice(t *testing.T) {
-	eCS := []string{"", ""}
-	if cs, err := ConfigSlice(" , "); err != nil {
-		t.Error("Unexpected error: ", err)
-	} else if !reflect.DeepEqual(eCS, cs) {
-		t.Errorf("Expecting: %v, received: %v", eCS, cs)
-	}
-}
-
-func TestParseCfgDerivedCharging(t *testing.T) {
-	eFieldsCfg := []byte(`[derived_charging]
-run_ids = run1, run2
-run_filters =,
-reqtype_fields = test1, test2 
-direction_fields = test1, test2
-tenant_fields = test1, test2
-category_fields = test1, test2
-account_fields = test1, test2
-subject_fields = test1, test2
-destination_fields = test1, test2
-setup_time_fields = test1, test2
-answer_time_fields = test1, test2
-usage_fields = test1, test2
-`)
-	edcs := utils.DerivedChargers{
-		&utils.DerivedCharger{RunId: "run1", ReqTypeField: "test1", DirectionField: "test1", TenantField: "test1", CategoryField: "test1",
-			AccountField: "test1", SubjectField: "test1", DestinationField: "test1", SetupTimeField: "test1", AnswerTimeField: "test1", UsageField: "test1"},
-		&utils.DerivedCharger{RunId: "run2", ReqTypeField: "test2", DirectionField: "test2", TenantField: "test2", CategoryField: "test2",
-			AccountField: "test2", SubjectField: "test2", DestinationField: "test2", SetupTimeField: "test2", AnswerTimeField: "test2", UsageField: "test2"}}
-	if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil {
-		t.Error("Could not parse the config", err.Error())
-	} else if !reflect.DeepEqual(cfg.DerivedChargers, edcs) {
-		t.Errorf("Expecting: %v, received: %v", edcs, cfg.DerivedChargers)
-	}
-}
-
-func TestParseCfgDerivedChargingDn1(t *testing.T) {
-	eFieldsCfg := []byte(`[derived_charging]
-run_ids = run1, run2
-run_filters =~account:s/^\w+[mpls]\d{6}$//,~account:s/^0\d{9}$//;^account/value/
-reqtype_fields = test1, test2 
-direction_fields = test1, test2
-tenant_fields = test1, test2
-category_fields = test1, test2
-account_fields = test1, test2
-subject_fields = test1, test2
-destination_fields = test1, test2
-setup_time_fields = test1, test2
-answer_time_fields = test1, test2
-usage_fields = test1, test2
-`)
-	eDcs := make(utils.DerivedChargers, 2)
-	if dc, err := utils.NewDerivedCharger("run1", `~account:s/^\w+[mpls]\d{6}$//`, "test1", "test1", "test1",
-		"test1", "test1", "test1", "test1", "test1", "test1", "test1"); err != nil {
-		t.Error("Unexpected error: ", err)
-	} else {
-		eDcs[0] = dc
-	}
-	if dc, err := utils.NewDerivedCharger("run2", `~account:s/^0\d{9}$//;^account/value/`, "test2", "test2", "test2",
-		"test2", "test2", "test2", "test2", "test2", "test2", "test2"); err != nil {
-		t.Error("Unexpected error: ", err)
-	} else {
-		eDcs[1] = dc
-	}
-
-	if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil {
-		t.Error("Could not parse the config", err.Error())
-	} else if !reflect.DeepEqual(cfg.DerivedChargers, eDcs) {
-		dcsJson, _ := json.Marshal(cfg.DerivedChargers)
-		t.Errorf("Received: %s", string(dcsJson))
-	}
-}
-
-func TestParseCdrcCdrFields(t *testing.T) {
-	eFieldsCfg := []byte(`[cdrc]
-cdr_type = test
-tor_field = tor1
-accid_field = accid1
-reqtype_field = reqtype1
-direction_field = direction1
-tenant_field = tenant1
-category_field = category1
-account_field = account1
-subject_field = subject1
-destination_field = destination1
-setup_time_field = setuptime1
-answer_time_field = answertime1
-usage_field = duration1
-extra_fields = extra1:extraval1,extra2:extraval1
-`)
-	eCdrcCdrFlds := map[string][]*utils.RSRField{
-		utils.TOR:         []*utils.RSRField{&utils.RSRField{Id: "tor1"}},
-		utils.ACCID:       []*utils.RSRField{&utils.RSRField{Id: "accid1"}},
-		utils.REQTYPE:     []*utils.RSRField{&utils.RSRField{Id: "reqtype1"}},
-		utils.DIRECTION:   []*utils.RSRField{&utils.RSRField{Id: "direction1"}},
-		utils.TENANT:      []*utils.RSRField{&utils.RSRField{Id: "tenant1"}},
-		utils.CATEGORY:    []*utils.RSRField{&utils.RSRField{Id: "category1"}},
-		utils.ACCOUNT:     []*utils.RSRField{&utils.RSRField{Id: "account1"}},
-		utils.SUBJECT:     []*utils.RSRField{&utils.RSRField{Id: "subject1"}},
-		utils.DESTINATION: []*utils.RSRField{&utils.RSRField{Id: "destination1"}},
-		utils.SETUP_TIME:  []*utils.RSRField{&utils.RSRField{Id: "setuptime1"}},
-		utils.ANSWER_TIME: []*utils.RSRField{&utils.RSRField{Id: "answertime1"}},
-		utils.USAGE:       []*utils.RSRField{&utils.RSRField{Id: "duration1"}},
-		"extra1":          []*utils.RSRField{&utils.RSRField{Id: "extraval1"}},
-		"extra2":          []*utils.RSRField{&utils.RSRField{Id: "extraval1"}},
-	}
-	if cfg, err := NewCGRConfigFromBytes(eFieldsCfg); err != nil {
-		t.Error("Could not parse the config", err.Error())
-	} else if !reflect.DeepEqual(cfg.CdrcCdrFields, eCdrcCdrFlds) {
-		t.Errorf("Expecting: %v, received: %v, tor: %v", eCdrcCdrFlds, cfg.CdrcCdrFields, cfg.CdrcCdrFields[utils.TOR])
-	}
-}

config/libconfig.go

@@ -0,0 +1,30 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"github.com/cgrates/cgrates/utils"
+)
+
+type CdrReplicationCfg struct {
+	Transport   string
+	Server      string
+	Synchronous bool
+	CdrFilter   utils.RSRFields // Only replicate if the filters here are matching
+}

config/libconfig_json.go

@@ -0,0 +1,243 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+// General config section
+type GeneralJsonCfg struct {
+	Http_skip_tls_verify *bool
+	Rounding_decimals    *int
+	Dbdata_encoding      *string
+	Tpexport_dir         *string
+	Default_reqtype      *string
+	Default_category     *string
+	Default_tenant       *string
+	Default_subject      *string
+	Reconnects           *int
+	Connect_attempts     *int
+}
+
+// Listen config section
+type ListenJsonCfg struct {
+	Rpc_json *string
+	Rpc_gob  *string
+	Http     *string
+}
+
+// Database config
+type DbJsonCfg struct {
+	Db_type        *string
+	Db_host        *string
+	Db_port        *int
+	Db_name        *string
+	Db_user        *string
+	Db_passwd      *string
+	Max_open_conns *int // Used only in case of storDb
+	Max_idle_conns *int
+}
+
+// Balancer config section
+type BalancerJsonCfg struct {
+	Enabled *bool
+}
+
+// Rater config section
+type RaterJsonCfg struct {
+	Enabled  *bool
+	Balancer *string
+	Cdrstats *string
+	Historys *string
+	Pubsubs  *string
+	Users    *string
+}
+
+// Scheduler config section
+type SchedulerJsonCfg struct {
+	Enabled *bool
+}
+
+// Cdrs config section
+type CdrsJsonCfg struct {
+	Enabled         *bool
+	Extra_fields    *[]string
+	Store_cdrs      *bool
+	Rater           *string
+	Cdrstats        *string
+	Reconnects      *int
+	Cdr_replication *[]*CdrReplicationJsonCfg
+}
+
+type CdrReplicationJsonCfg struct {
+	Transport   *string
+	Server      *string
+	Synchronous *bool
+	Cdr_filter  *string
+}
+
+// Cdrstats config section
+type CdrStatsJsonCfg struct {
+	Enabled       *bool
+	Save_Interval *string
+}
+
+// One cdr field config, used in cdre and cdrc
+type CdrFieldJsonCfg struct {
+	Tag          *string
+	Type         *string
+	Cdr_field_id *string
+	Metatag_id   *string
+	Value        *string
+	Width        *int
+	Strip        *string
+	Padding      *string
+	Layout       *string
+	Field_filter *string
+	Mandatory    *bool
+}
+
+// Cdre config section
+type CdreJsonCfg struct {
+	Cdr_format                    *string
+	Field_separator               *string
+	Data_usage_multiply_factor    *float64
+	Sms_usage_multiply_factor     *float64
+	Generic_usage_multiply_factor *float64
+	Cost_multiply_factor          *float64
+	Cost_rounding_decimals        *int
+	Cost_shift_digits             *int
+	Mask_destination_id           *string
+	Mask_length                   *int
+	Export_dir                    *string
+	Header_fields                 *[]*CdrFieldJsonCfg
+	Content_fields                *[]*CdrFieldJsonCfg
+	Trailer_fields                *[]*CdrFieldJsonCfg
+}
+
+// Cdrc config section
+type CdrcJsonCfg struct {
+	Enabled                    *bool
+	Dry_run                    *bool
+	Cdrs                       *string
+	Cdr_format                 *string
+	Field_separator            *string
+	Run_delay                  *int
+	Data_usage_multiply_factor *float64
+	Cdr_in_dir                 *string
+	Cdr_out_dir                *string
+	Failed_calls_prefix        *string
+	Cdr_source_id              *string
+	Cdr_filter                 *string
+	Max_open_files             *int
+	Partial_record_cache       *string
+	Header_fields              *[]*CdrFieldJsonCfg
+	Content_fields             *[]*CdrFieldJsonCfg
+	Trailer_fields             *[]*CdrFieldJsonCfg
+}
+
+// SM-FreeSWITCH config section
+type SmFsJsonCfg struct {
+	Enabled                *bool
+	Rater                  *string
+	Cdrs                   *string
+	Reconnects             *int
+	Create_cdr             *bool
+	Cdr_extra_fields       *[]string
+	Debit_interval         *string
+	Min_call_duration      *string
+	Max_call_duration      *string
+	Min_dur_low_balance    *string
+	Low_balance_ann_file   *string
+	Empty_balance_context  *string
+	Empty_balance_ann_file *string
+	Subscribe_park         *bool
+	Channel_sync_interval  *string
+	Connections            *[]*FsConnJsonCfg
+}
+
+// Represents one connection instance towards FreeSWITCH
+type FsConnJsonCfg struct {
+	Server     *string
+	Password   *string
+	Reconnects *int
+}
+
+// SM-Kamailio config section
+type SmKamJsonCfg struct {
+	Enabled           *bool
+	Rater             *string
+	Cdrs              *string
+	Reconnects        *int
+	Create_cdr        *bool
+	Debit_interval    *string
+	Min_call_duration *string
+	Max_call_duration *string
+	Connections       *[]*KamConnJsonCfg
+}
+
+// Represents one connection instance towards Kamailio
+type KamConnJsonCfg struct {
+	Evapi_addr *string
+	Reconnects *int
+}
+
+// SM-OpenSIPS config section
+type SmOsipsJsonCfg struct {
+	Enabled                   *bool
+	Listen_udp                *string
+	Rater                     *string
+	Cdrs                      *string
+	Reconnects                *int
+	Create_cdr                *bool
+	Debit_interval            *string
+	Min_call_duration         *string
+	Max_call_duration         *string
+	Events_subscribe_interval *string
+	Mi_addr                   *string
+}
+
+// Represents one connection instance towards OpenSIPS
+type OsipsConnJsonCfg struct {
+	Mi_addr    *string
+	Reconnects *int
+}
+
+// History server config section
+type HistServJsonCfg struct {
+	Enabled       *bool
+	History_dir   *string
+	Save_interval *string
+}
+
+// PubSub server config section
+type PubSubServJsonCfg struct {
+	Enabled *bool
+}
+
+// PubSub server config section
+type UserServJsonCfg struct {
+	Enabled *bool
+	Indexes *[]string
+}
+
+// Mailer config section
+type MailerJsonCfg struct {
+	Server       *string
+	Auth_user    *string
+	Auth_passwd  *string
+	From_address *string
+}

config/multifiles_local_test.go

@@ -0,0 +1,106 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"flag"
+	"github.com/cgrates/cgrates/utils"
+	"testing"
+)
+
+var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, disabled by default.") // This flag will be passed here via "go test -local" args
+
+var mfCgrCfg *CGRConfig
+
+func TestMfInitConfig(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	if mfCgrCfg, err = NewCGRConfigFromFolder("/usr/share/cgrates/conf/samples/multifiles"); err != nil {
+		t.Fatal("Got config error: ", err.Error())
+	}
+}
+
+func TestMfGeneralItems(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if mfCgrCfg.DefaultReqType != utils.META_PSEUDOPREPAID { // Twice reconfigured
+		t.Error("DefaultReqType: ", mfCgrCfg.DefaultReqType)
+	}
+	if mfCgrCfg.DefaultCategory != "call" { // Not configred, should be inherited from default
+		t.Error("DefaultCategory: ", mfCgrCfg.DefaultCategory)
+	}
+}
+
+func TestMfCdreDefaultInstance(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	for _, prflName := range []string{"*default", "export1"} {
+		if _, hasIt := mfCgrCfg.CdreProfiles[prflName]; !hasIt {
+			t.Error("Cdre does not contain profile ", prflName)
+		}
+	}
+	prfl := "*default"
+	if mfCgrCfg.CdreProfiles[prfl].CdrFormat != "csv" {
+		t.Error("Default instance has cdrFormat: ", mfCgrCfg.CdreProfiles[prfl].CdrFormat)
+	}
+	if mfCgrCfg.CdreProfiles[prfl].DataUsageMultiplyFactor != 1024.0 {
+		t.Error("Default instance has cdrFormat: ", mfCgrCfg.CdreProfiles[prfl].DataUsageMultiplyFactor)
+	}
+	if len(mfCgrCfg.CdreProfiles[prfl].HeaderFields) != 0 {
+		t.Error("Default instance has number of header fields: ", len(mfCgrCfg.CdreProfiles[prfl].HeaderFields))
+	}
+	if len(mfCgrCfg.CdreProfiles[prfl].ContentFields) != 12 {
+		t.Error("Default instance has number of content fields: ", len(mfCgrCfg.CdreProfiles[prfl].ContentFields))
+	}
+	if mfCgrCfg.CdreProfiles[prfl].ContentFields[2].Tag != "Direction" {
+		t.Error("Unexpected headerField value: ", mfCgrCfg.CdreProfiles[prfl].ContentFields[2].Tag)
+	}
+}
+
+func TestMfCdreExport1Instance(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	prfl := "export1"
+	if mfCgrCfg.CdreProfiles[prfl].CdrFormat != "csv" {
+		t.Error("Export1 instance has cdrFormat: ", mfCgrCfg.CdreProfiles[prfl].CdrFormat)
+	}
+	if mfCgrCfg.CdreProfiles[prfl].DataUsageMultiplyFactor != 1.0 {
+		t.Error("Export1 instance has DataUsageMultiplyFormat: ", mfCgrCfg.CdreProfiles[prfl].DataUsageMultiplyFactor)
+	}
+	if mfCgrCfg.CdreProfiles[prfl].CostRoundingDecimals != 3.0 {
+		t.Error("Export1 instance has cdrFormat: ", mfCgrCfg.CdreProfiles[prfl].CostRoundingDecimals)
+	}
+	if len(mfCgrCfg.CdreProfiles[prfl].HeaderFields) != 2 {
+		t.Error("Export1 instance has number of header fields: ", len(mfCgrCfg.CdreProfiles[prfl].HeaderFields))
+	}
+	if mfCgrCfg.CdreProfiles[prfl].HeaderFields[1].Tag != "RunId" {
+		t.Error("Unexpected headerField value: ", mfCgrCfg.CdreProfiles[prfl].HeaderFields[1].Tag)
+	}
+	if len(mfCgrCfg.CdreProfiles[prfl].ContentFields) != 9 {
+		t.Error("Export1 instance has number of content fields: ", len(mfCgrCfg.CdreProfiles[prfl].ContentFields))
+	}
+	if mfCgrCfg.CdreProfiles[prfl].ContentFields[2].Tag != "Account" {
+		t.Error("Unexpected headerField value: ", mfCgrCfg.CdreProfiles[prfl].ContentFields[2].Tag)
+	}
+}

config/smconfig.go

@@ -0,0 +1,311 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"github.com/cgrates/cgrates/utils"
+	"time"
+)
+
+// Returns the first cached default value for a SM-FreeSWITCH connection
+func NewDfltFsConnConfig() *FsConnConfig {
+	if dfltFsConnConfig == nil {
+		return new(FsConnConfig) // No defaults, most probably we are building the defaults now
+	}
+	dfltVal := *dfltFsConnConfig // Copy the value instead of it's pointer
+	return &dfltVal
+}
+
+// One connection to FreeSWITCH server
+type FsConnConfig struct {
+	Server     string
+	Password   string
+	Reconnects int
+}
+
+func (self *FsConnConfig) loadFromJsonCfg(jsnCfg *FsConnJsonCfg) error {
+	if jsnCfg == nil {
+		return nil
+	}
+	if jsnCfg.Server != nil {
+		self.Server = *jsnCfg.Server
+	}
+	if jsnCfg.Password != nil {
+		self.Password = *jsnCfg.Password
+	}
+	if jsnCfg.Reconnects != nil {
+		self.Reconnects = *jsnCfg.Reconnects
+	}
+	return nil
+}
+
+type SmFsConfig struct {
+	Enabled             bool
+	Rater               string
+	Cdrs                string
+	Reconnects          int
+	CreateCdr           bool
+	CdrExtraFields      []*utils.RSRField
+	DebitInterval       time.Duration
+	MinCallDuration     time.Duration
+	MaxCallDuration     time.Duration
+	MinDurLowBalance    time.Duration
+	LowBalanceAnnFile   string
+	EmptyBalanceContext string
+	EmptyBalanceAnnFile string
+	SubscribePark       bool
+	ChannelSyncInterval time.Duration
+	Connections         []*FsConnConfig
+}
+
+func (self *SmFsConfig) loadFromJsonCfg(jsnCfg *SmFsJsonCfg) error {
+	if jsnCfg == nil {
+		return nil
+	}
+	var err error
+	if jsnCfg.Enabled != nil {
+		self.Enabled = *jsnCfg.Enabled
+	}
+	if jsnCfg.Rater != nil {
+		self.Rater = *jsnCfg.Rater
+	}
+	if jsnCfg.Cdrs != nil {
+		self.Cdrs = *jsnCfg.Cdrs
+	}
+	if jsnCfg.Reconnects != nil {
+		self.Reconnects = *jsnCfg.Reconnects
+	}
+	if jsnCfg.Create_cdr != nil {
+		self.CreateCdr = *jsnCfg.Create_cdr
+	}
+	if jsnCfg.Cdr_extra_fields != nil {
+		if self.CdrExtraFields, err = utils.ParseRSRFieldsFromSlice(*jsnCfg.Cdr_extra_fields); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Debit_interval != nil {
+		if self.DebitInterval, err = utils.ParseDurationWithSecs(*jsnCfg.Debit_interval); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Min_call_duration != nil {
+		if self.MinCallDuration, err = utils.ParseDurationWithSecs(*jsnCfg.Min_call_duration); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Max_call_duration != nil {
+		if self.MaxCallDuration, err = utils.ParseDurationWithSecs(*jsnCfg.Max_call_duration); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Min_dur_low_balance != nil {
+		if self.MinDurLowBalance, err = utils.ParseDurationWithSecs(*jsnCfg.Min_dur_low_balance); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Low_balance_ann_file != nil {
+		self.LowBalanceAnnFile = *jsnCfg.Low_balance_ann_file
+	}
+	if jsnCfg.Empty_balance_context != nil {
+		self.EmptyBalanceContext = *jsnCfg.Empty_balance_context
+	}
+	if jsnCfg.Empty_balance_ann_file != nil {
+		self.EmptyBalanceAnnFile = *jsnCfg.Empty_balance_ann_file
+	}
+	if jsnCfg.Subscribe_park != nil {
+		self.SubscribePark = *jsnCfg.Subscribe_park
+	}
+	if jsnCfg.Channel_sync_interval != nil {
+		if self.ChannelSyncInterval, err = utils.ParseDurationWithSecs(*jsnCfg.Channel_sync_interval); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Connections != nil {
+		self.Connections = make([]*FsConnConfig, len(*jsnCfg.Connections))
+		for idx, jsnConnCfg := range *jsnCfg.Connections {
+			self.Connections[idx] = NewDfltFsConnConfig()
+			self.Connections[idx].loadFromJsonCfg(jsnConnCfg)
+		}
+	}
+	return nil
+}
+
+// Returns the first cached default value for a SM-FreeSWITCH connection
+func NewDfltKamConnConfig() *KamConnConfig {
+	if dfltKamConnConfig == nil {
+		return new(KamConnConfig) // No defaults, most probably we are building the defaults now
+	}
+	dfltVal := *dfltKamConnConfig
+	return &dfltVal
+}
+
+// Represents one connection instance towards Kamailio
+type KamConnConfig struct {
+	EvapiAddr  string
+	Reconnects int
+}
+
+func (self *KamConnConfig) loadFromJsonCfg(jsnCfg *KamConnJsonCfg) error {
+	if jsnCfg == nil {
+		return nil
+	}
+	if jsnCfg.Evapi_addr != nil {
+		self.EvapiAddr = *jsnCfg.Evapi_addr
+	}
+	if jsnCfg.Reconnects != nil {
+		self.Reconnects = *jsnCfg.Reconnects
+	}
+	return nil
+}
+
+// SM-Kamailio config section
+type SmKamConfig struct {
+	Enabled         bool
+	Rater           string
+	Cdrs            string
+	Reconnects      int
+	CreateCdr       bool
+	DebitInterval   time.Duration
+	MinCallDuration time.Duration
+	MaxCallDuration time.Duration
+	Connections     []*KamConnConfig
+}
+
+func (self *SmKamConfig) loadFromJsonCfg(jsnCfg *SmKamJsonCfg) error {
+	if jsnCfg == nil {
+		return nil
+	}
+	var err error
+	if jsnCfg.Enabled != nil {
+		self.Enabled = *jsnCfg.Enabled
+	}
+	if jsnCfg.Rater != nil {
+		self.Rater = *jsnCfg.Rater
+	}
+	if jsnCfg.Cdrs != nil {
+		self.Cdrs = *jsnCfg.Cdrs
+	}
+	if jsnCfg.Reconnects != nil {
+		self.Reconnects = *jsnCfg.Reconnects
+	}
+	if jsnCfg.Create_cdr != nil {
+		self.CreateCdr = *jsnCfg.Create_cdr
+	}
+	if jsnCfg.Debit_interval != nil {
+		if self.DebitInterval, err = utils.ParseDurationWithSecs(*jsnCfg.Debit_interval); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Min_call_duration != nil {
+		if self.MinCallDuration, err = utils.ParseDurationWithSecs(*jsnCfg.Min_call_duration); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Max_call_duration != nil {
+		if self.MaxCallDuration, err = utils.ParseDurationWithSecs(*jsnCfg.Max_call_duration); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Connections != nil {
+		self.Connections = make([]*KamConnConfig, len(*jsnCfg.Connections))
+		for idx, jsnConnCfg := range *jsnCfg.Connections {
+			self.Connections[idx] = NewDfltKamConnConfig()
+			self.Connections[idx].loadFromJsonCfg(jsnConnCfg)
+		}
+	}
+	return nil
+}
+
+// Represents one connection instance towards OpenSIPS, not in use for now but planned for future
+type OsipsConnConfig struct {
+	MiAddr     string
+	Reconnects int
+}
+
+func (self *OsipsConnConfig) loadFromJsonCfg(jsnCfg *OsipsConnJsonCfg) error {
+	if jsnCfg.Mi_addr != nil {
+		self.MiAddr = *jsnCfg.Mi_addr
+	}
+	if jsnCfg.Reconnects != nil {
+		self.Reconnects = *jsnCfg.Reconnects
+	}
+	return nil
+}
+
+// SM-OpenSIPS config section
+type SmOsipsConfig struct {
+	Enabled                 bool
+	ListenUdp               string
+	Rater                   string
+	Cdrs                    string
+	Reconnects              int
+	CreateCdr               bool
+	DebitInterval           time.Duration
+	MinCallDuration         time.Duration
+	MaxCallDuration         time.Duration
+	EventsSubscribeInterval time.Duration
+	MiAddr                  string
+}
+
+func (self *SmOsipsConfig) loadFromJsonCfg(jsnCfg *SmOsipsJsonCfg) error {
+	var err error
+	if jsnCfg.Enabled != nil {
+		self.Enabled = *jsnCfg.Enabled
+	}
+	if jsnCfg.Listen_udp != nil {
+		self.ListenUdp = *jsnCfg.Listen_udp
+	}
+	if jsnCfg.Rater != nil {
+		self.Rater = *jsnCfg.Rater
+	}
+	if jsnCfg.Cdrs != nil {
+		self.Cdrs = *jsnCfg.Cdrs
+	}
+	if jsnCfg.Reconnects != nil {
+		self.Reconnects = *jsnCfg.Reconnects
+	}
+	if jsnCfg.Create_cdr != nil {
+		self.CreateCdr = *jsnCfg.Create_cdr
+	}
+	if jsnCfg.Debit_interval != nil {
+		if self.DebitInterval, err = utils.ParseDurationWithSecs(*jsnCfg.Debit_interval); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Min_call_duration != nil {
+		if self.MinCallDuration, err = utils.ParseDurationWithSecs(*jsnCfg.Min_call_duration); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Max_call_duration != nil {
+		if self.MaxCallDuration, err = utils.ParseDurationWithSecs(*jsnCfg.Max_call_duration); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Events_subscribe_interval != nil {
+		if self.EventsSubscribeInterval, err = utils.ParseDurationWithSecs(*jsnCfg.Events_subscribe_interval); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Mi_addr != nil {
+		self.MiAddr = *jsnCfg.Mi_addr
+	}
+
+	return nil
+}

config/smconfig_test.go

@@ -0,0 +1,59 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"github.com/cgrates/cgrates/utils"
+	"reflect"
+	"testing"
+)
+
+func TesSmFsConfigLoadFromJsonCfg(t *testing.T) {
+	smFsJsnCfg := &SmFsJsonCfg{
+		Enabled:        utils.BoolPointer(true),
+		Create_cdr:     utils.BoolPointer(true),
+		Subscribe_park: utils.BoolPointer(true),
+		Connections: &[]*FsConnJsonCfg{
+			&FsConnJsonCfg{
+				Server:     utils.StringPointer("1.2.3.4:8021"),
+				Password:   utils.StringPointer("ClueCon"),
+				Reconnects: utils.IntPointer(5),
+			},
+			&FsConnJsonCfg{
+				Server:     utils.StringPointer("2.3.4.5:8021"),
+				Password:   utils.StringPointer("ClueCon"),
+				Reconnects: utils.IntPointer(5),
+			},
+		},
+	}
+	eSmFsConfig := &SmFsConfig{Enabled: true,
+		CreateCdr:     true,
+		SubscribePark: true,
+		Connections: []*FsConnConfig{
+			&FsConnConfig{Server: "1.2.3.4:8021", Password: "ClueCon", Reconnects: 5},
+			&FsConnConfig{Server: "1.2.3.4:8021", Password: "ClueCon", Reconnects: 5},
+		},
+	}
+	smFsCfg := new(SmFsConfig)
+	if err := smFsCfg.loadFromJsonCfg(smFsJsnCfg); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eSmFsConfig, smFsCfg) {
+		t.Error("Received: ", smFsCfg)
+	}
+}