Adding Account and User attributes to tutorial/Users.csv, removing .Debug through code

Fix account command in the docs

.gitignore

@@ -12,3 +12,4 @@ bin
 dean*
 data/vagrant/.vagrant
 data/vagrant/vagrant_ansible_inventory_default
+data/tutorials/fs_evsock/freeswitch/etc/freeswitch/

.travis.yml

@@ -1,7 +1,7 @@
 language: go
 
 go:
-  - 1.2
+  - 1.4
 
 script: $TRAVIS_BUILD_DIR/test.sh
 

CONTRIBUTING.md

@@ -0,0 +1,26 @@
+## Contributor Agreement (required for pull requests)
+
+While the CGRateS project is open source under the GPLv3 license, contributions
+back to this project which contain code must be made available to ITsysCOM GmbH under the terms of the MIT license.
+
+### What does this mean?
+
+This means ITsysCOM GmbH can do the following with code contributed
+back to the project from the open source community.
+
+1. Include the code in future CGRateS releases.
+2. Include the code, in original or modified form, in other products created by ITsysCOM GmbH, **including closed-source products**.
+
+### How do I agree to these terms?
+
+To agree to these terms, sign the contributing agreement in `CONTRIBUTORS.md` and
+include the change as a commit with your pull request. **Pull requests which contain
+commits from users not explicitly listed in the `CONTRIBUTORS.md` file will be
+rejected *without review*.**
+
+* You only need to sign the `CONTRIBUTORS.md` file prior to the first time you
+  submit a pull request (or include it as part of the pull request itself).
+* The submitter of a pull request which adds a user to the `CONTRIBUTORS.md` file
+  *must* match the user information which was added to the `CONTRIBUTORS.md` file.
+  For example, only @danbogos can send a pull request containing a commit which adds
+  @danbogos to the `CONTRIBUTORS.md` file.

CONTRIBUTORS.md

@@ -0,0 +1,42 @@
+## Contributors
+
+By signing this agreement, the undersigned agree that contributions made to the CGRateS
+project are provided under the terms of the MIT license, included below. For additional
+information, please see the [`CONTRIBUTING.md`](CONTRIBUTING.md) file.
+
+**Pull requests which add a user to this file may only be submitted by the user who was added to the file.**
+
+### Contributor License Text
+
+> The MIT License (MIT)
+>
+> Copyright © <year> <copyright holders>
+>
+> Permission is hereby granted, free of charge, to any person obtaining a copy
+> of this software and associated documentation files (the "Software"), to deal
+> in the Software without restriction, including without limitation the rights
+> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+> copies of the Software, and to permit persons to whom the Software is
+> furnished to do so, subject to the following conditions:
+>
+> The above copyright notice and this permission notice shall be included in
+> all copies or substantial portions of the Software.
+>
+> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+> THE SOFTWARE.
+
+### Undersigned
+
+| GitHub Username | Full Name |
+| --------------- | --------- |
+| @danbogos | Dan Christian Bogos |
+| @eloycoto | Eloy Coto Pereiro |
+
+<!-- to sign, include a single line above this comment containing the following text:
+| @username | First Last |
+-->

README.md

@@ -1,6 +1,6 @@
-## Rating system for Telecom & ISP environments ##
+## Real-time Charging System for Telecom & ISP environments ##
 
-[![Build Status](https://drone.io/github.com/cgrates/cgrates/status.png)](https://drone.io/github.com/cgrates/cgrates/latest) [![Build Status](https://secure.travis-ci.org/cgrates/cgrates.png)](http://travis-ci.org/cgrates/cgrates)
+[![Build Status](https://secure.travis-ci.org/cgrates/cgrates.png)](http://travis-ci.org/cgrates/cgrates)
 
 ### Features ###
 + Rates for prepaid and for postpaid
@@ -24,4 +24,5 @@ PDF, Epub, Manpage http://readthedocs.org/projects/cgrates/downloads/
 
 API reference [godoc](http://godoc.org/github.com/cgrates/cgrates/apier)
 
-Also check irc.freenode.net#cgrates and [Google group](https://groups.google.com/forum/#!forum/cgrates) for a more real-time support.
+Also check [irc.freenode.net #cgrates](irc://irc.freenode.net:6667/cgrates)  ([Webchat](http://webchat.freenode.net?randomnick=1&channels=%23cgrates)) and [Google group](https://groups.google.com/forum/#!forum/cgrates) for a more real-time support.
+

apier/apier.go

@@ -1,648 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package apier
-
-import (
-	"errors"
-	"fmt"
-	"path"
-	"time"
-
-	"github.com/cgrates/cgrates/cache2go"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/scheduler"
-	"github.com/cgrates/cgrates/utils"
-)
-
-const (
-	OK = "OK"
-)
-
-type ApierV1 struct {
-	StorDb    engine.LoadStorage
-	RatingDb  engine.RatingStorage
-	AccountDb engine.AccountingStorage
-	CdrDb     engine.CdrStorage
-	LogDb     engine.LogStorage
-	Sched     *scheduler.Scheduler
-	Config    *config.CGRConfig
-}
-
-func (self *ApierV1) GetDestination(dstId string, reply *engine.Destination) error {
-	if dst, err := self.RatingDb.GetDestination(dstId); err != nil {
-		return errors.New(utils.ERR_NOT_FOUND)
-	} else {
-		*reply = *dst
-	}
-	return nil
-}
-
-func (self *ApierV1) GetRatingPlan(rplnId string, reply *engine.RatingPlan) error {
-	if rpln, err := self.RatingDb.GetRatingPlan(rplnId, false); err != nil {
-		return errors.New(utils.ERR_NOT_FOUND)
-	} else {
-		*reply = *rpln
-	}
-	return nil
-}
-
-type AttrGetAccount struct {
-	Tenant      string
-	Account     string
-	BalanceType string
-	Direction   string
-}
-
-// Get balance
-func (self *ApierV1) GetAccount(attr *AttrGetAccount, reply *engine.Account) error {
-	tag := fmt.Sprintf("%s:%s:%s", attr.Direction, attr.Tenant, attr.Account)
-	userBalance, err := self.AccountDb.GetAccount(tag)
-	if err != nil {
-		return err
-	}
-
-	*reply = *userBalance
-	return nil
-}
-
-type AttrAddBalance struct {
-	Tenant         string
-	Account        string
-	BalanceType    string
-	Direction      string
-	Value          float64
-	ExpirationDate time.Time
-	RatingSubject  string
-	DestinationId  string
-	Weight         float64
-	Overwrite      bool // When true it will reset if the balance is already there
-}
-
-func (self *ApierV1) AddBalance(attr *AttrAddBalance, reply *string) error {
-	tag := fmt.Sprintf("%s:%s:%s", attr.Direction, attr.Tenant, attr.Account)
-	if _, err := self.AccountDb.GetAccount(tag); err != nil {
-		// create user balance if not exists
-		ub := &engine.Account{
-			Id: tag,
-		}
-		if err := self.AccountDb.SetAccount(ub); err != nil {
-			*reply = err.Error()
-			return err
-		}
-	}
-	at := &engine.ActionTiming{
-		AccountIds: []string{tag},
-	}
-
-	if attr.Direction == "" {
-		attr.Direction = engine.OUTBOUND
-	}
-	aType := engine.TOPUP
-	if attr.Overwrite {
-		aType = engine.TOPUP_RESET
-	}
-	at.SetActions(engine.Actions{
-		&engine.Action{
-			ActionType:  aType,
-			BalanceType: attr.BalanceType,
-			Direction:   attr.Direction,
-			Balance: &engine.Balance{
-				Value:          attr.Value,
-				ExpirationDate: attr.ExpirationDate,
-				RateSubject:    attr.RatingSubject,
-				DestinationId:  attr.DestinationId,
-				Weight:         attr.Weight,
-			},
-		},
-	})
-	if err := at.Execute(); err != nil {
-		*reply = err.Error()
-		return err
-	}
-	*reply = OK
-	return nil
-}
-
-type AttrExecuteAction struct {
-	Direction string
-	Tenant    string
-	Account   string
-	ActionsId string
-}
-
-func (self *ApierV1) ExecuteAction(attr *AttrExecuteAction, reply *string) error {
-	tag := fmt.Sprintf("%s:%s:%s", attr.Direction, attr.Tenant, attr.Account)
-	at := &engine.ActionTiming{
-		AccountIds: []string{tag},
-		ActionsId:  attr.ActionsId,
-	}
-
-	if err := at.Execute(); err != nil {
-		*reply = err.Error()
-		return err
-	}
-	*reply = OK
-	return nil
-}
-
-type AttrLoadRatingPlan struct {
-	TPid         string
-	RatingPlanId string
-}
-
-// Process dependencies and load a specific rating plan from storDb into dataDb.
-func (self *ApierV1) LoadRatingPlan(attrs AttrLoadRatingPlan, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingPlanId"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	dbReader := engine.NewDbReader(self.StorDb, self.RatingDb, self.AccountDb, attrs.TPid)
-	if loaded, err := dbReader.LoadRatingPlanByTag(attrs.RatingPlanId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if !loaded {
-		return errors.New("NOT_FOUND")
-	}
-	//Automatic cache of the newly inserted rating plan
-	didNotChange := []string{}
-	if err := self.RatingDb.CacheRating(nil, nil, didNotChange, didNotChange); err != nil {
-		return err
-	}
-	*reply = OK
-	return nil
-}
-
-// Process dependencies and load a specific rating profile from storDb into dataDb.
-func (self *ApierV1) LoadRatingProfile(attrs utils.TPRatingProfile, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "TOR", "Direction", "Subject"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	dbReader := engine.NewDbReader(self.StorDb, self.RatingDb, self.AccountDb, attrs.TPid)
-	if err := dbReader.LoadRatingProfileFiltered(&attrs); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	//Automatic cache of the newly inserted rating profile
-	didNotChange := []string{}
-	if err := self.RatingDb.CacheRating(didNotChange, didNotChange, []string{engine.RATING_PROFILE_PREFIX + attrs.KeyId()}, didNotChange); err != nil {
-		return err
-	}
-	*reply = OK
-	return nil
-}
-
-type AttrSetRatingProfile struct {
-	Tenant                string                      // Tenant's Id
-	TOR                   string                      // TypeOfRecord
-	Direction             string                      // Traffic direction, OUT is the only one supported for now
-	Subject               string                      // Rating subject, usually the same as account
-	Overwrite             bool                        // Overwrite if exists
-	RatingPlanActivations []*utils.TPRatingActivation // Activate rating plans at specific time
-}
-
-// Sets a specific rating profile working with data directly in the RatingDb without involving storDb
-func (self *ApierV1) SetRatingProfile(attrs AttrSetRatingProfile, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "TOR", "Direction", "Subject", "RatingPlanActivations"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	for _, rpa := range attrs.RatingPlanActivations {
-		if missing := utils.MissingStructFields(rpa, []string{"ActivationTime", "RatingPlanId"}); len(missing) != 0 {
-			return fmt.Errorf("%s:RatingPlanActivation:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-		}
-	}
-	tpRpf := utils.TPRatingProfile{Tenant: attrs.Tenant, TOR: attrs.TOR, Direction: attrs.Direction, Subject: attrs.Subject}
-	keyId := tpRpf.KeyId()
-	if !attrs.Overwrite {
-		if exists, err := self.RatingDb.HasData(engine.RATING_PROFILE_PREFIX, keyId); err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-		} else if exists {
-			return errors.New(utils.ERR_EXISTS)
-		}
-	}
-	rpfl := &engine.RatingProfile{Id: keyId, RatingPlanActivations: make(engine.RatingPlanActivations, len(attrs.RatingPlanActivations))}
-	for idx, ra := range attrs.RatingPlanActivations {
-		at, err := utils.ParseDate(ra.ActivationTime)
-		if err != nil {
-			return fmt.Errorf(fmt.Sprintf("%s:Cannot parse activation time from %v", utils.ERR_SERVER_ERROR, ra.ActivationTime))
-		}
-		if exists, err := self.RatingDb.HasData(engine.RATING_PLAN_PREFIX, ra.RatingPlanId); err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-		} else if !exists {
-			return fmt.Errorf(fmt.Sprintf("%s:RatingPlanId:%s", utils.ERR_NOT_FOUND, ra.RatingPlanId))
-		}
-		rpfl.RatingPlanActivations[idx] = &engine.RatingPlanActivation{ActivationTime: at, RatingPlanId: ra.RatingPlanId,
-			FallbackKeys: utils.FallbackSubjKeys(tpRpf.Direction, tpRpf.Tenant, tpRpf.TOR, ra.FallbackSubjects)}
-	}
-	if err := self.RatingDb.SetRatingProfile(rpfl); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	//Automatic cache of the newly inserted rating profile
-	didNotChange := []string{}
-	if err := self.RatingDb.CacheRating(didNotChange, didNotChange, []string{engine.RATING_PROFILE_PREFIX + keyId}, didNotChange); err != nil {
-		return err
-	}
-	*reply = OK
-	return nil
-}
-
-type AttrSetActions struct {
-	ActionsId string            // Actions id
-	Overwrite bool              // If previously defined, will be overwritten
-	Actions   []*utils.TPAction // Set of actions this Actions profile will perform
-}
-
-func (self *ApierV1) SetActions(attrs AttrSetActions, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"ActionsId", "Actions"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	for _, action := range attrs.Actions {
-		requiredFields := []string{"Identifier", "Weight"}
-		if action.BalanceType != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions
-			requiredFields = append(requiredFields, "Direction", "Units")
-		}
-		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
-			return fmt.Errorf("%s:Action:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
-		}
-	}
-	if !attrs.Overwrite {
-		if exists, err := self.AccountDb.HasData(engine.ACTION_PREFIX, attrs.ActionsId); err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-		} else if exists {
-			return errors.New(utils.ERR_EXISTS)
-		}
-	}
-	storeActions := make(engine.Actions, len(attrs.Actions))
-	for idx, apiAct := range attrs.Actions {
-		a := &engine.Action{
-			Id:               utils.GenUUID(),
-			ActionType:       apiAct.Identifier,
-			BalanceType:      apiAct.BalanceType,
-			Direction:        apiAct.Direction,
-			Weight:           apiAct.Weight,
-			ExpirationString: apiAct.ExpiryTime,
-			ExtraParameters:  apiAct.ExtraParameters,
-			Balance: &engine.Balance{
-				Uuid:          utils.GenUUID(),
-				Value:         apiAct.Units,
-				Weight:        apiAct.BalanceWeight,
-				DestinationId: apiAct.DestinationId,
-				RateSubject:   apiAct.RatingSubject,
-			},
-		}
-		storeActions[idx] = a
-	}
-	if err := self.AccountDb.SetActions(attrs.ActionsId, storeActions); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	*reply = OK
-	return nil
-}
-
-type AttrSetActionPlan struct {
-	Id              string             // Profile id
-	ActionPlan      []*ApiActionTiming // Set of actions this Actions profile will perform
-	Overwrite       bool               // If previously defined, will be overwritten
-	ReloadScheduler bool               // Enables automatic reload of the scheduler (eg: useful when adding a single action timing)
-}
-
-type ApiActionTiming struct {
-	ActionsId string  // Actions id
-	Years     string  // semicolon separated list of years this timing is valid on, *any or empty supported
-	Months    string  // semicolon separated list of months this timing is valid on, *any or empty supported
-	MonthDays string  // semicolon separated list of month's days this timing is valid on, *any or empty supported
-	WeekDays  string  // semicolon separated list of week day names this timing is valid on *any or empty supported
-	Time      string  // String representing the time this timing starts on, *asap supported
-	Weight    float64 // Binding's weight
-}
-
-func (self *ApierV1) SetActionPlan(attrs AttrSetActionPlan, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"Id", "ActionPlan"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	for _, at := range attrs.ActionPlan {
-		requiredFields := []string{"ActionsId", "Time", "Weight"}
-		if missing := utils.MissingStructFields(at, requiredFields); len(missing) != 0 {
-			return fmt.Errorf("%s:Action:%s:%v", utils.ERR_MANDATORY_IE_MISSING, at.ActionsId, missing)
-		}
-	}
-	if !attrs.Overwrite {
-		if exists, err := self.AccountDb.HasData(engine.ACTION_TIMING_PREFIX, attrs.Id); err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-		} else if exists {
-			return errors.New(utils.ERR_EXISTS)
-		}
-	}
-	storeAtms := make(engine.ActionPlan, len(attrs.ActionPlan))
-	for idx, apiAtm := range attrs.ActionPlan {
-		if exists, err := self.AccountDb.HasData(engine.ACTION_PREFIX, apiAtm.ActionsId); err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-		} else if !exists {
-			return fmt.Errorf("%s:%s", utils.ERR_BROKEN_REFERENCE, err.Error())
-		}
-		timing := new(engine.RITiming)
-		timing.Years.Parse(apiAtm.Years, ";")
-		timing.Months.Parse(apiAtm.Months, ";")
-		timing.MonthDays.Parse(apiAtm.MonthDays, ";")
-		timing.WeekDays.Parse(apiAtm.WeekDays, ";")
-		timing.StartTime = apiAtm.Time
-		at := &engine.ActionTiming{
-			Uuid:      utils.GenUUID(),
-			Id:        attrs.Id,
-			Weight:    apiAtm.Weight,
-			Timing:    &engine.RateInterval{Timing: timing},
-			ActionsId: apiAtm.ActionsId,
-		}
-		storeAtms[idx] = at
-	}
-	if err := self.AccountDb.SetActionTimings(attrs.Id, storeAtms); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	if attrs.ReloadScheduler {
-		if self.Sched == nil {
-			return errors.New("SCHEDULER_NOT_ENABLED")
-		}
-		self.Sched.LoadActionTimings(self.AccountDb)
-		self.Sched.Restart()
-	}
-	*reply = OK
-	return nil
-}
-
-type AttrAddActionTrigger struct {
-	Tenant         string
-	Account        string
-	Direction      string
-	BalanceType    string
-	ThresholdType  string
-	ThresholdValue float64
-	DestinationId  string
-	Weight         float64
-	ActionsId      string
-}
-
-func (self *ApierV1) AddTriggeredAction(attr AttrAddActionTrigger, reply *string) error {
-	if attr.Direction == "" {
-		attr.Direction = engine.OUTBOUND
-	}
-
-	at := &engine.ActionTrigger{
-		Id:             utils.GenUUID(),
-		BalanceType:    attr.BalanceType,
-		Direction:      attr.Direction,
-		ThresholdType:  attr.ThresholdType,
-		ThresholdValue: attr.ThresholdValue,
-		DestinationId:  attr.DestinationId,
-		Weight:         attr.Weight,
-		ActionsId:      attr.ActionsId,
-		Executed:       false,
-	}
-
-	tag := utils.BalanceKey(attr.Tenant, attr.Account, attr.Direction)
-	_, err := engine.AccLock.Guard(tag, func() (float64, error) {
-		userBalance, err := self.AccountDb.GetAccount(tag)
-		if err != nil {
-			return 0, err
-		}
-
-		userBalance.ActionTriggers = append(userBalance.ActionTriggers, at)
-
-		if err = self.AccountDb.SetAccount(userBalance); err != nil {
-			return 0, err
-		}
-		return 0, nil
-	})
-	if err != nil {
-		*reply = err.Error()
-		return err
-	}
-	*reply = OK
-	return nil
-}
-
-// Process dependencies and load a specific AccountActions profile from storDb into dataDb.
-func (self *ApierV1) LoadAccountActions(attrs utils.TPAccountActions, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "Account", "Direction"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	dbReader := engine.NewDbReader(self.StorDb, self.RatingDb, self.AccountDb, attrs.TPid)
-
-	if _, err := engine.AccLock.Guard(attrs.KeyId(), func() (float64, error) {
-		if err := dbReader.LoadAccountActionsFiltered(&attrs); err != nil {
-			return 0, err
-		}
-		return 0, nil
-	}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	// ToDo: Get the action keys loaded by dbReader so we reload only these in cache
-	// Need to do it before scheduler otherwise actions to run will be unknown
-	if err := self.AccountDb.CacheAccounting(nil, nil, nil); err != nil {
-		return err
-	}
-	if self.Sched != nil {
-		self.Sched.LoadActionTimings(self.AccountDb)
-		self.Sched.Restart()
-	}
-	*reply = OK
-	return nil
-}
-
-func (self *ApierV1) ReloadScheduler(input string, reply *string) error {
-	if self.Sched == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
-	}
-	self.Sched.LoadActionTimings(self.AccountDb)
-	self.Sched.Restart()
-	*reply = OK
-	return nil
-
-}
-
-func (self *ApierV1) ReloadCache(attrs utils.ApiReloadCache, reply *string) error {
-	var dstKeys, rpKeys, rpfKeys, actKeys, shgKeys, rpAlsKeys, accAlsKeys []string
-	if len(attrs.DestinationIds) > 0 {
-		dstKeys = make([]string, len(attrs.DestinationIds))
-		for idx, dId := range attrs.DestinationIds {
-			dstKeys[idx] = engine.DESTINATION_PREFIX + dId // Cache expects them as redis keys
-		}
-	}
-	if len(attrs.RatingPlanIds) > 0 {
-		rpKeys = make([]string, len(attrs.RatingPlanIds))
-		for idx, rpId := range attrs.RatingPlanIds {
-			rpKeys[idx] = engine.RATING_PLAN_PREFIX + rpId
-		}
-	}
-	if len(attrs.RatingProfileIds) > 0 {
-		rpfKeys = make([]string, len(attrs.RatingProfileIds))
-		for idx, rpfId := range attrs.RatingProfileIds {
-			rpfKeys[idx] = engine.RATING_PROFILE_PREFIX + rpfId
-		}
-	}
-	if len(attrs.ActionIds) > 0 {
-		actKeys = make([]string, len(attrs.ActionIds))
-		for idx, actId := range attrs.ActionIds {
-			actKeys[idx] = engine.ACTION_PREFIX + actId
-		}
-	}
-	if len(attrs.SharedGroupIds) > 0 {
-		shgKeys = make([]string, len(attrs.SharedGroupIds))
-		for idx, shgId := range attrs.SharedGroupIds {
-			shgKeys[idx] = engine.SHARED_GROUP_PREFIX + shgId
-		}
-	}
-	if len(attrs.RpAliases) > 0 {
-		rpAlsKeys = make([]string, len(attrs.RpAliases))
-		for idx, alias := range attrs.RpAliases {
-			rpAlsKeys[idx] = engine.RP_ALIAS_PREFIX + alias
-		}
-	}
-	if len(attrs.AccAliases) > 0 {
-		accAlsKeys = make([]string, len(attrs.AccAliases))
-		for idx, alias := range attrs.AccAliases {
-			accAlsKeys[idx] = engine.ACC_ALIAS_PREFIX + alias
-		}
-	}
-	if err := self.RatingDb.CacheRating(dstKeys, rpKeys, rpfKeys, rpAlsKeys); err != nil {
-		return err
-	}
-	if err := self.AccountDb.CacheAccounting(actKeys, shgKeys, accAlsKeys); err != nil {
-		return err
-	}
-	*reply = "OK"
-	return nil
-}
-
-func (self *ApierV1) GetCacheStats(attrs utils.AttrCacheStats, reply *utils.CacheStats) error {
-	cs := new(utils.CacheStats)
-	cs.Destinations = cache2go.CountEntries(engine.DESTINATION_PREFIX)
-	cs.RatingPlans = cache2go.CountEntries(engine.RATING_PLAN_PREFIX)
-	cs.RatingProfiles = cache2go.CountEntries(engine.RATING_PROFILE_PREFIX)
-	cs.Actions = cache2go.CountEntries(engine.ACTION_PREFIX)
-	cs.SharedGroups = cache2go.CountEntries(engine.SHARED_GROUP_PREFIX)
-	cs.RatingAliases = cache2go.CountEntries(engine.RP_ALIAS_PREFIX)
-	cs.AccountAliases = cache2go.CountEntries(engine.ACC_ALIAS_PREFIX)
-	*reply = *cs
-	return nil
-}
-
-func (self *ApierV1) GetCachedItemAge(itemId string, reply *utils.CachedItemAge) error {
-	if len(itemId) == 0 {
-		return fmt.Errorf("%s:ItemId", utils.ERR_MANDATORY_IE_MISSING)
-	}
-	cachedItemAge := new(utils.CachedItemAge)
-	var found bool
-	for idx, cacheKey := range []string{engine.DESTINATION_PREFIX + itemId, engine.RATING_PLAN_PREFIX + itemId, engine.RATING_PROFILE_PREFIX + itemId,
-		engine.ACTION_PREFIX + itemId, engine.SHARED_GROUP_PREFIX + itemId, engine.RP_ALIAS_PREFIX + itemId, engine.ACC_ALIAS_PREFIX + itemId} {
-		if age, err := cache2go.GetKeyAge(cacheKey); err == nil {
-			found = true
-			switch idx {
-			case 0:
-				cachedItemAge.Destination = age
-			case 1:
-				cachedItemAge.RatingPlan = age
-			case 2:
-				cachedItemAge.RatingProfile = age
-			case 3:
-				cachedItemAge.Action = age
-			case 4:
-				cachedItemAge.SharedGroup = age
-			case 5:
-				cachedItemAge.RatingAlias = age
-			case 6:
-				cachedItemAge.AccountAlias = age
-			}
-		}
-	}
-	if !found {
-		return errors.New(utils.ERR_NOT_FOUND)
-	}
-	*reply = *cachedItemAge
-	return nil
-}
-
-func (self *ApierV1) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder, reply *string) error {
-	loader := engine.NewFileCSVReader(self.RatingDb, self.AccountDb, utils.CSV_SEP,
-		path.Join(attrs.FolderPath, utils.DESTINATIONS_CSV),
-		path.Join(attrs.FolderPath, utils.TIMINGS_CSV),
-		path.Join(attrs.FolderPath, utils.RATES_CSV),
-		path.Join(attrs.FolderPath, utils.DESTINATION_RATES_CSV),
-		path.Join(attrs.FolderPath, utils.RATING_PLANS_CSV),
-		path.Join(attrs.FolderPath, utils.RATING_PROFILES_CSV),
-		path.Join(attrs.FolderPath, utils.SHARED_GROUPS_CSV),
-		path.Join(attrs.FolderPath, utils.ACTIONS_CSV),
-		path.Join(attrs.FolderPath, utils.ACTION_PLANS_CSV),
-		path.Join(attrs.FolderPath, utils.ACTION_TRIGGERS_CSV),
-		path.Join(attrs.FolderPath, utils.ACCOUNT_ACTIONS_CSV))
-	if err := loader.LoadAll(); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	if attrs.DryRun {
-		*reply = "OK"
-		return nil // Mission complete, no errors
-	}
-	if err := loader.WriteToDatabase(attrs.FlushDb, false); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	// Make sure the items are in the cache
-	dstIds, _ := loader.GetLoadedIds(engine.DESTINATION_PREFIX)
-	dstKeys := make([]string, len(dstIds))
-	for idx, dId := range dstIds {
-		dstKeys[idx] = engine.DESTINATION_PREFIX + dId // Cache expects them as redis keys
-	}
-	rplIds, _ := loader.GetLoadedIds(engine.RATING_PLAN_PREFIX)
-	rpKeys := make([]string, len(rplIds))
-	for idx, rpId := range rplIds {
-		rpKeys[idx] = engine.RATING_PLAN_PREFIX + rpId
-	}
-	rpfIds, _ := loader.GetLoadedIds(engine.RATING_PROFILE_PREFIX)
-	rpfKeys := make([]string, len(rpfIds))
-	for idx, rpfId := range rpfIds {
-		rpfKeys[idx] = engine.RATING_PROFILE_PREFIX + rpfId
-	}
-	actIds, _ := loader.GetLoadedIds(engine.ACTION_PREFIX)
-	actKeys := make([]string, len(actIds))
-	for idx, actId := range actIds {
-		actKeys[idx] = engine.ACTION_PREFIX + actId
-	}
-	shgIds, _ := loader.GetLoadedIds(engine.SHARED_GROUP_PREFIX)
-	shgKeys := make([]string, len(shgIds))
-	for idx, shgId := range shgIds {
-		shgKeys[idx] = engine.SHARED_GROUP_PREFIX + shgId
-	}
-	rpAliases, _ := loader.GetLoadedIds(engine.RP_ALIAS_PREFIX)
-	rpAlsKeys := make([]string, len(rpAliases))
-	for idx, alias := range rpAliases {
-		rpAlsKeys[idx] = engine.RP_ALIAS_PREFIX + alias
-	}
-	accAliases, _ := loader.GetLoadedIds(engine.ACC_ALIAS_PREFIX)
-	accAlsKeys := make([]string, len(accAliases))
-	for idx, alias := range accAliases {
-		accAlsKeys[idx] = engine.ACC_ALIAS_PREFIX + alias
-	}
-	if err := self.RatingDb.CacheRating(dstKeys, rpKeys, rpfKeys, rpAlsKeys); err != nil {
-		return err
-	}
-	if err := self.AccountDb.CacheAccounting(actKeys, shgKeys, accAlsKeys); err != nil {
-		return err
-	}
-	if self.Sched != nil {
-		self.Sched.LoadActionTimings(self.AccountDb)
-		self.Sched.Restart()
-	}
-	*reply = "OK"
-	return nil
-}

apier/cdre.go

@@ -1,153 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package apier
-
-import (
-	"fmt"
-	"github.com/cgrates/cgrates/cdre"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/utils"
-	"os"
-	"path"
-	"strconv"
-	"strings"
-	"time"
-)
-
-// Export Cdrs to file
-func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.ExportedFileCdrs) error {
-	var tStart, tEnd time.Time
-	var err error
-	cdrFormat := strings.ToLower(attr.CdrFormat)
-	if !utils.IsSliceMember(utils.CdreCdrFormats, cdrFormat) {
-		return fmt.Errorf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, "CdrFormat")
-	}
-	if len(attr.TimeStart) != 0 {
-		if tStart, err = utils.ParseTimeDetectLayout(attr.TimeStart); err != nil {
-			return err
-		}
-	}
-	if len(attr.TimeEnd) != 0 {
-		if tEnd, err = utils.ParseTimeDetectLayout(attr.TimeEnd); err != nil {
-			return err
-		}
-	}
-	fileName := attr.ExportFileName
-	exportId := attr.ExportId
-	if len(exportId) == 0 {
-		exportId = strconv.FormatInt(time.Now().Unix(), 10)
-	}
-	roundDecimals := attr.RoundingDecimals
-	if roundDecimals == 0 {
-		roundDecimals = self.Config.RoundingDecimals
-	}
-	cdrs, err := self.CdrDb.GetStoredCdrs(attr.CgrIds, attr.MediationRunId, attr.CdrHost, attr.CdrSource, attr.ReqType, attr.Direction,
-		attr.Tenant, attr.Tor, attr.Account, attr.Subject, attr.DestinationPrefix, tStart, tEnd, attr.SkipErrors, attr.SkipRated)
-	if err != nil {
-		return err
-	} else if len(cdrs) == 0 {
-		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
-		return nil
-	}
-	switch cdrFormat {
-	case utils.CDRE_DRYRUN:
-		exportedIds := make([]string, len(cdrs))
-		for idxCdr, cdr := range cdrs {
-			exportedIds[idxCdr] = cdr.CgrId
-		}
-		*reply = utils.ExportedFileCdrs{ExportedFilePath: utils.CDRE_DRYRUN, TotalRecords: len(cdrs), ExportedCgrIds: exportedIds}
-	case utils.CDRE_CSV:
-		if len(fileName) == 0 {
-			fileName = fmt.Sprintf("cdre_%s.csv", exportId)
-		}
-		exportedFields := self.Config.CdreExportedFields
-		if len(attr.ExportTemplate) != 0 {
-			if exportedFields, err = config.ParseRSRFields(attr.ExportTemplate); err != nil {
-				return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-			}
-		}
-		if len(exportedFields) == 0 {
-			return fmt.Errorf("%s:ExportTemplate", utils.ERR_MANDATORY_IE_MISSING)
-		}
-		filePath := path.Join(self.Config.CdreDir, fileName)
-		fileOut, err := os.Create(filePath)
-		if err != nil {
-			return err
-		}
-		defer fileOut.Close()
-		csvWriter := cdre.NewCsvCdrWriter(fileOut, roundDecimals, exportedFields)
-		exportedIds := make([]string, 0)
-		unexportedIds := make(map[string]string)
-		for _, cdr := range cdrs {
-			if err := csvWriter.WriteCdr(cdr); err != nil {
-				unexportedIds[cdr.CgrId] = err.Error()
-			} else {
-				exportedIds = append(exportedIds, cdr.CgrId)
-			}
-		}
-		csvWriter.Close()
-		*reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), ExportedCgrIds: exportedIds, UnexportedCgrIds: unexportedIds}
-	case utils.CDRE_FIXED_WIDTH:
-		if len(fileName) == 0 {
-			fileName = fmt.Sprintf("cdre_%s.fwv", exportId)
-		}
-		exportTemplate := self.Config.CdreFWXmlTemplate
-		if len(attr.ExportTemplate) != 0 && self.Config.XmlCfgDocument != nil {
-			if xmlTemplate, err := self.Config.XmlCfgDocument.GetCdreFWCfg(attr.ExportTemplate[len(utils.XML_PROFILE_PREFIX):]); err != nil {
-				return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-			} else if xmlTemplate != nil {
-				exportTemplate = xmlTemplate
-			}
-		}
-		if exportTemplate == nil {
-			return fmt.Errorf("%s:ExportTemplate", utils.ERR_MANDATORY_IE_MISSING)
-		}
-		filePath := path.Join(self.Config.CdreDir, fileName)
-		fileOut, err := os.Create(filePath)
-		if err != nil {
-			return err
-		}
-		defer fileOut.Close()
-		fww, _ := cdre.NewFWCdrWriter(self.LogDb, fileOut, exportTemplate, exportId, roundDecimals)
-		exportedIds := make([]string, 0)
-		unexportedIds := make(map[string]string)
-		for _, cdr := range cdrs {
-			if err := fww.WriteCdr(cdr); err != nil {
-				unexportedIds[cdr.CgrId] = err.Error()
-			} else {
-				exportedIds = append(exportedIds, cdr.CgrId)
-			}
-		}
-		fww.Close()
-		*reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), ExportedCgrIds: exportedIds, UnexportedCgrIds: unexportedIds}
-	}
-	return nil
-}
-
-// Remove Cdrs out of CDR storage
-func (self *ApierV1) RemCdrs(attrs utils.AttrRemCdrs, reply *string) error {
-	if len(attrs.CgrIds) == 0 {
-		return fmt.Errorf("%s:CgrIds", utils.ERR_MANDATORY_IE_MISSING)
-	}
-	if err := self.CdrDb.RemStoredCdrs(attrs.CgrIds); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	*reply = "OK"
-	return nil
-}

apier/cdrs.go

@@ -1,45 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package apier
-
-import (
-	"fmt"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
-)
-
-type AttrGetCallCost struct {
-	CgrId string // Unique id of the CDR
-	RunId string // Run Id
-}
-
-// Retrieves the callCost out of CGR logDb
-func (apier *ApierV1) GetCallCostLog(attrs AttrGetCallCost, reply *engine.CallCost) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"CgrId", "RunId"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if cc, err := apier.LogDb.GetCallCostLog(attrs.CgrId, "", attrs.RunId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if cc == nil {
-		return fmt.Errorf("NOT_FOUND")
-	} else {
-		*reply = *cc
-	}
-	return nil
-}

apier/tpaccountactions.go

@@ -1,104 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package apier
-
-import (
-	"errors"
-	"fmt"
-
-	"github.com/cgrates/cgrates/utils"
-)
-
-// Creates a new AccountActions profile within a tariff plan
-func (self *ApierV1) SetTPAccountActions(attrs utils.TPAccountActions, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs,
-		[]string{"TPid", "LoadId", "Tenant", "Account", "Direction", "ActionPlanId", "ActionTriggersId"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if err := self.StorDb.SetTPAccountActions(attrs.TPid, map[string]*utils.TPAccountActions{attrs.KeyId(): &attrs}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	*reply = "OK"
-	return nil
-}
-
-type AttrGetTPAccountActions struct {
-	TPid   string // Tariff plan id
-	LoadId string // AccountActions id
-
-}
-
-// Queries specific AccountActions profile on tariff plan
-func (self *ApierV1) GetTPAccountActions(attrs utils.TPAccountActions, reply *[]*utils.TPAccountActions) error {
-	mndtryFlds := []string{"TPid", "LoadId"}
-	if len(attrs.Account) != 0 { // If account provided as filter, make all related fields mandatory
-		mndtryFlds = append(mndtryFlds, "Tenant", "Account", "Direction")
-	}
-	if missing := utils.MissingStructFields(&attrs, mndtryFlds); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if aa, err := self.StorDb.GetTpAccountActions(&attrs); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if len(aa) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
-	} else {
-		var acts []*utils.TPAccountActions
-		if len(attrs.Account) != 0 {
-			acts = []*utils.TPAccountActions{aa[attrs.KeyId()]}
-		} else {
-			for _, actLst := range aa {
-				acts = append(acts, actLst)
-			}
-		}
-		*reply = acts
-	}
-	return nil
-}
-
-type AttrGetTPAccountActionIds struct {
-	TPid string // Tariff plan id
-}
-
-// Queries AccountActions identities on specific tariff plan.
-func (self *ApierV1) GetTPAccountActionLoadIds(attrs AttrGetTPAccountActionIds, reply *[]string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_ACCOUNT_ACTIONS, "loadid", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
-	} else {
-		*reply = ids
-	}
-	return nil
-}
-
-// Removes specific AccountActions on Tariff plan
-func (self *ApierV1) RemTPAccountActions(attrs utils.TPAccountActions, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "Account", "Direction"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_ACCOUNT_ACTIONS, attrs.TPid, attrs.LoadId, attrs.Tenant, attrs.Account, attrs.Direction); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else {
-		*reply = "OK"
-	}
-	return nil
-}

apier/tpratingprofiles.go

@@ -1,105 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package apier
-
-// This file deals with tp_rate_profiles management over APIs
-
-import (
-	"errors"
-	"fmt"
-
-	"github.com/cgrates/cgrates/utils"
-)
-
-// Creates a new RatingProfile within a tariff plan
-func (self *ApierV1) SetTPRatingProfile(attrs utils.TPRatingProfile, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "TOR", "Direction", "Subject", "RatingPlanActivations"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if err := self.StorDb.SetTPRatingProfiles(attrs.TPid, map[string]*utils.TPRatingProfile{attrs.KeyId(): &attrs}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	}
-	*reply = "OK"
-	return nil
-}
-
-type AttrGetTPRatingProfile struct {
-	TPid   string // Tariff plan id
-	LoadId string // RatingProfile id
-}
-
-// Queries specific RatingProfile on tariff plan
-func (self *ApierV1) GetTPRatingProfiles(attrs utils.TPRatingProfile, reply *[]*utils.TPRatingProfile) error {
-	mndtryFlds := []string{"TPid", "LoadId"}
-	if len(attrs.Subject) != 0 { // If Subject provided as filter, make all related fields mandatory
-		mndtryFlds = append(mndtryFlds, "Tenant", "TOR", "Direction", "Subject")
-	}
-	if missing := utils.MissingStructFields(&attrs, mndtryFlds); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if dr, err := self.StorDb.GetTpRatingProfiles(&attrs); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if dr == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
-	} else {
-		var rpfs []*utils.TPRatingProfile
-		if len(attrs.Subject) != 0 {
-			rpfs = []*utils.TPRatingProfile{dr[attrs.KeyId()]}
-		} else {
-			for _, rpfLst := range dr {
-				rpfs = append(rpfs, rpfLst)
-			}
-		}
-		*reply = rpfs
-	}
-	return nil
-}
-
-// Queries RatingProfile identities on specific tariff plan.
-func (self *ApierV1) GetTPRatingProfileLoadIds(attrs utils.AttrTPRatingProfileIds, reply *[]string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_RATE_PROFILES, "loadid", map[string]string{
-		"tenant":    attrs.Tenant,
-		"tor":       attrs.TOR,
-		"direction": attrs.Direction,
-		"subject":   attrs.Subject,
-	}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
-	} else {
-		*reply = ids
-	}
-	return nil
-}
-
-// Removes specific RatingProfile on Tariff plan
-func (self *ApierV1) RemTPRatingProfile(attrs utils.TPRatingProfile, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "TOR", "Direction", "Subject"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
-	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_RATE_PROFILES, attrs.TPid, attrs.LoadId, attrs.Tenant, attrs.TOR, attrs.Direction, attrs.Subject); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else {
-		*reply = "OK"
-	}
-	return nil
-}

apier/tutfscsv_local_test.go

@@ -1,285 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package apier
-
-import (
-	"fmt"
-	"net/rpc/jsonrpc"
-	"os"
-	"os/exec"
-	"path"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
-)
-
-var fscsvCfgPath string
-var fscsvCfg *config.CGRConfig
-
-func init() {
-	fscsvCfgPath = path.Join(*dataDir, "tutorials", "fs_csv", "cgrates", "etc", "cgrates", "cgrates.cfg")
-	fscsvCfg, _ = config.NewCGRConfig(&fscsvCfgPath)
-}
-
-// Remove here so they can be properly created by init script
-func TestFsCsvRemoveDirs(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	for _, pathDir := range []string{cfg.CdreDir, cfg.CdrcCdrInDir, cfg.CdrcCdrOutDir, cfg.HistoryDir} {
-		if err := os.RemoveAll(pathDir); err != nil {
-			t.Fatal("Error removing folder: ", pathDir, err)
-		}
-	}
-}
-
-// Empty tables before using them
-func TestFsCsvCreateTables(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	if *storDbType != utils.MYSQL {
-		t.Fatal("Unsupported storDbType")
-	}
-	var mysql *engine.MySQLStorage
-	if d, err := engine.NewMySQLStorage(fscsvCfg.StorDBHost, fscsvCfg.StorDBPort, fscsvCfg.StorDBName, fscsvCfg.StorDBUser, fscsvCfg.StorDBPass); err != nil {
-		t.Fatal("Error on opening database connection: ", err)
-	} else {
-		mysql = d.(*engine.MySQLStorage)
-	}
-	for _, scriptName := range []string{engine.CREATE_CDRS_TABLES_SQL, engine.CREATE_COSTDETAILS_TABLES_SQL, engine.CREATE_MEDIATOR_TABLES_SQL, engine.CREATE_TARIFFPLAN_TABLES_SQL} {
-		if err := mysql.CreateTablesFromScript(path.Join(*dataDir, "storage", *storDbType, scriptName)); err != nil {
-			t.Fatal("Error on mysql creation: ", err.Error())
-			return // No point in going further
-		}
-	}
-	for _, tbl := range []string{utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_EXTRA} {
-		if _, err := mysql.Db.Query(fmt.Sprintf("SELECT 1 from %s", tbl)); err != nil {
-			t.Fatal(err.Error())
-		}
-	}
-}
-
-func TestFsCsvInitDataDb(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	ratingDb, err := engine.ConfigureRatingStorage(fscsvCfg.RatingDBType, fscsvCfg.RatingDBHost, fscsvCfg.RatingDBPort, fscsvCfg.RatingDBName, fscsvCfg.RatingDBUser, fscsvCfg.RatingDBPass, fscsvCfg.DBDataEncoding)
-	if err != nil {
-		t.Fatal("Cannot connect to dataDb", err)
-	}
-	accountDb, err := engine.ConfigureAccountingStorage(fscsvCfg.AccountDBType, fscsvCfg.AccountDBHost, fscsvCfg.AccountDBPort, fscsvCfg.AccountDBName,
-		fscsvCfg.AccountDBUser, fscsvCfg.AccountDBPass, fscsvCfg.DBDataEncoding)
-	if err != nil {
-		t.Fatal("Cannot connect to dataDb", err)
-	}
-	for _, db := range []engine.Storage{ratingDb, accountDb} {
-		if err := db.Flush(); err != nil {
-			t.Fatal("Cannot reset dataDb", err)
-		}
-	}
-}
-
-func TestFsCsvStartFs(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	exec.Command("pkill", "freeswitch").Run() // Just to make sure no freeswitch is running
-	go func() {
-		fs := exec.Command("sudo", "/usr/share/cgrates/tutorials/fs_csv/freeswitch/etc/init.d/freeswitch", "start")
-		out, _ := fs.CombinedOutput()
-		engine.Logger.Info(fmt.Sprintf("CgrEngine-TestFsCsv: %s", out))
-	}()
-	time.Sleep(time.Duration(*waitFs) * time.Millisecond) // Give time to rater to fire up
-}
-
-// Finds cgr-engine executable and starts it with default configuration
-func TestFsCsvStartEngine(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	exec.Command("pkill", "cgr-engine").Run() // Just to make sure another one is not running, bit brutal maybe we can fine tune it
-	go func() {
-		eng := exec.Command("sudo", "/usr/share/cgrates/tutorials/fs_json/cgrates/etc/init.d/cgrates", "start")
-		out, _ := eng.CombinedOutput()
-		engine.Logger.Info(fmt.Sprintf("CgrEngine-TestFsCsv: %s", out))
-	}()
-	time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time to rater to fire up
-}
-
-// Connect rpc client to rater
-func TestFsCsvRpcConn(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	var err error
-	rater, err = jsonrpc.Dial("tcp", fscsvCfg.RPCJSONListen)
-	if err != nil {
-		t.Fatal("Could not connect to rater: ", err.Error())
-	}
-}
-
-// Make sure we start with fresh data
-func TestFsCsvEmptyCache(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	var rcvStats *utils.CacheStats
-	expectedStats := &utils.CacheStats{Destinations: 0, RatingPlans: 0, RatingProfiles: 0, Actions: 0}
-	var args utils.AttrCacheStats
-	if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil {
-		t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())
-	} else if !reflect.DeepEqual(expectedStats, rcvStats) {
-		t.Errorf("Calling ApierV1.GetCacheStats expected: %v, received: %v", expectedStats, rcvStats)
-	}
-}
-
-func TestFsCsvLoadTariffPlans(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	reply := ""
-	// Simple test that command is executed without errors
-	attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tutorials", "fs_csv", "cgrates", "tariffplans")}
-	if err := rater.Call("ApierV1.LoadTariffPlanFromFolder", attrs, &reply); err != nil {
-		t.Error("Got error on ApierV1.LoadTariffPlanFromFolder: ", err.Error())
-	} else if reply != "OK" {
-		t.Error("Calling ApierV1.LoadTariffPlanFromFolder got reply: ", reply)
-	}
-	time.Sleep(100 * time.Millisecond) // Give time for scheduler to execute topups
-	var rcvStats *utils.CacheStats
-	expectedStats := &utils.CacheStats{Destinations: 3, RatingPlans: 1, RatingProfiles: 1, Actions: 2}
-	var args utils.AttrCacheStats
-	if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil {
-		t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())
-	} else if !reflect.DeepEqual(expectedStats, rcvStats) {
-		t.Errorf("Calling ApierV1.GetCacheStats expected: %v, received: %v", expectedStats, rcvStats)
-	}
-}
-
-func TestFsCsvGetAccount(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	var reply *engine.Account
-	attrs := &AttrGetAccount{Tenant: "cgrates.org", Account: "1001", BalanceType: "*monetary", Direction: "*out"}
-	if err := rater.Call("ApierV1.GetAccount", attrs, &reply); err != nil {
-		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
-	} else if reply.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue() != 10.0 { // We expect 11.5 since we have added in the previous test 1.5
-		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", reply.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue())
-	}
-}
-
-func TestFsCsvCall1(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	tStart := time.Date(2014, 01, 15, 6, 0, 0, 0, time.UTC)
-	tEnd := time.Date(2014, 01, 15, 6, 0, 35, 0, time.UTC)
-	cd := engine.CallDescriptor{
-		Direction:    "*out",
-		TOR:          "call",
-		Tenant:       "cgrates.org",
-		Subject:      "1001",
-		Account:      "1001",
-		Destination:  "1002",
-		TimeStart:    tStart,
-		TimeEnd:      tEnd,
-		CallDuration: 35,
-	}
-	var cc engine.CallCost
-	// Make sure the cost is what we expect it is
-	if err := rater.Call("Responder.GetCost", cd, &cc); err != nil {
-		t.Error("Got error on Responder.GetCost: ", err.Error())
-	} else if cc.GetConnectFee() != 0.4 && cc.Cost != 0.6 {
-		t.Errorf("Calling Responder.GetCost got callcost: %v", cc)
-	}
-	// Make sure debit charges what cost returned
-	if err := rater.Call("Responder.MaxDebit", cd, &cc); err != nil {
-		t.Error("Got error on Responder.MaxDebit: ", err.Error())
-	} else if cc.GetConnectFee() != 0.4 && cc.Cost != 0.6 {
-		t.Errorf("Calling Responder.MaxDebit got callcost: %v", cc)
-	}
-	// Make sure the account was debited correctly for the first loop index (ConnectFee included)
-	var reply *engine.Account
-	attrs := &AttrGetAccount{Tenant: "cgrates.org", Account: "1001", BalanceType: "*monetary", Direction: "*out"}
-	if err := rater.Call("ApierV1.GetAccount", attrs, &reply); err != nil {
-		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
-	} else if reply.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue() != 9.4 { // We expect 11.5 since we have added in the previous test 1.5
-		t.Errorf("Calling ApierV1.GetAccount expected: 9.4, received: %f", reply.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue())
-	} else if len(reply.UnitCounters) != 1 ||
-		utils.Round(reply.UnitCounters[0].Balances[0].Value, 2, utils.ROUNDING_MIDDLE) != 0.6 { // Make sure we correctly count usage
-		t.Errorf("Received unexpected UnitCounters: %v", reply.UnitCounters)
-	}
-	cd = engine.CallDescriptor{
-		Direction:    "*out",
-		TOR:          "call",
-		Tenant:       "cgrates.org",
-		Subject:      "1001",
-		Account:      "1001",
-		Destination:  "1002",
-		TimeStart:    tStart,
-		TimeEnd:      tEnd,
-		CallDuration: 35,
-		LoopIndex:    1, // Should not charge ConnectFee
-	}
-	// Make sure debit charges what cost returned
-	if err := rater.Call("Responder.MaxDebit", cd, &cc); err != nil {
-		t.Error("Got error on Responder.MaxDebit: ", err.Error())
-	} else if cc.GetConnectFee() != 0.4 && cc.Cost != 0.2 { // Does not contain connectFee, however connectFee should be correctly reported
-		t.Errorf("Calling Responder.MaxDebit got callcost: %v", cc)
-	}
-	// Make sure the account was debited correctly for the first loop index (ConnectFee included)
-	var reply2 *engine.Account
-	if err := rater.Call("ApierV1.GetAccount", attrs, &reply2); err != nil {
-		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
-	} else if utils.Round(reply2.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue(), 2, utils.ROUNDING_MIDDLE) != 9.20 {
-		t.Errorf("Calling ApierV1.GetAccount expected: 9.2, received: %f", reply2.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue())
-	} else if len(reply2.UnitCounters) != 1 ||
-		utils.Round(reply2.UnitCounters[0].Balances[0].Value, 2, utils.ROUNDING_MIDDLE) != 0.8 { // Make sure we correctly count usage
-		t.Errorf("Received unexpected UnitCounters: %v", reply2.UnitCounters)
-	}
-}
-
-// Simply kill the engine after we are done with tests within this file
-func TestFsCsvStopEngine(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	go func() {
-		eng := exec.Command("/usr/share/cgrates/tutorials/fs_csv/cgrates/etc/init.d/cgrates", "stop")
-		out, _ := eng.CombinedOutput()
-		engine.Logger.Info(fmt.Sprintf("CgrEngine-TestFsCsv: %s", out))
-	}()
-}
-
-func TestFsCsvStopFs(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	go func() {
-		fs := exec.Command("/usr/share/cgrates/tutorials/fs_csv/freeswitch/etc/init.d/freeswitch", "stop")
-		out, _ := fs.CombinedOutput()
-		engine.Logger.Info(fmt.Sprintf("CgrEngine-TestFsCsv: %s", out))
-	}()
-}

apier/v1/accounts.go

@@ -1,6 +1,6 @@
 /*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,11 +16,12 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"errors"
-	"fmt"
+	"math"
+	"regexp"
+	"strings"
 	"time"
 
 	"github.com/cgrates/cgrates/engine"
@@ -42,16 +43,16 @@ type AccountActionTiming struct {
 
 func (self *ApierV1) GetAccountActionPlan(attrs AttrAcntAction, reply *[]*AccountActionTiming) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Direction"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(strings.Join(missing, ","), "")
 	}
 	accountATs := make([]*AccountActionTiming, 0)
-	allATs, err := self.AccountDb.GetAllActionTimings()
+	allATs, err := self.RatingDb.GetAllActionPlans()
 	if err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	}
 	for _, ats := range allATs {
 		for _, at := range ats {
-			if utils.IsSliceMember(at.AccountIds, utils.BalanceKey(attrs.Tenant, attrs.Account, attrs.Direction)) {
+			if utils.IsSliceMember(at.AccountIds, utils.AccountKey(attrs.Tenant, attrs.Account, attrs.Direction)) {
 				accountATs = append(accountATs, &AccountActionTiming{Uuid: at.Uuid, ActionPlanId: at.Id, ActionsId: at.ActionsId, NextExecTime: at.GetNextStartTime(time.Now())})
 			}
 		}
@@ -72,31 +73,31 @@ type AttrRemActionTiming struct {
 // Removes an ActionTimings or parts of it depending on filters being set
 func (self *ApierV1) RemActionTiming(attrs AttrRemActionTiming, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"ActionPlanId"}); len(missing) != 0 { // Only mandatory ActionPlanId
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	if len(attrs.Account) != 0 { // Presence of Account requires complete account details to be provided
 		if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Direction"}); len(missing) != 0 {
-			return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+			return utils.NewErrMandatoryIeMissing(missing...)
 		}
 	}
-	_, err := engine.AccLock.Guard(engine.ACTION_TIMING_PREFIX, func() (float64, error) {
-		ats, err := self.AccountDb.GetActionTimings(attrs.ActionPlanId)
+	_, err := engine.Guardian.Guard(func() (interface{}, error) {
+		ats, err := self.RatingDb.GetActionPlans(attrs.ActionPlanId)
 		if err != nil {
 			return 0, err
 		} else if len(ats) == 0 {
-			return 0, errors.New(utils.ERR_NOT_FOUND)
+			return 0, utils.ErrNotFound
 		}
-		ats = engine.RemActionTiming(ats, attrs.ActionTimingId, utils.BalanceKey(attrs.Tenant, attrs.Account, attrs.Direction))
-		if err := self.AccountDb.SetActionTimings(attrs.ActionPlanId, ats); err != nil {
+		ats = engine.RemActionPlan(ats, attrs.ActionTimingId, utils.AccountKey(attrs.Tenant, attrs.Account, attrs.Direction))
+		if err := self.RatingDb.SetActionPlans(attrs.ActionPlanId, ats); err != nil {
 			return 0, err
 		}
 		return 0, nil
-	})
+	}, utils.ACTION_TIMING_PREFIX)
 	if err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	}
 	if attrs.ReloadScheduler && self.Sched != nil {
-		self.Sched.LoadActionTimings(self.AccountDb)
+		self.Sched.LoadActionPlans(self.RatingDb)
 		self.Sched.Restart()
 	}
 	*reply = OK
@@ -106,10 +107,10 @@ func (self *ApierV1) RemActionTiming(attrs AttrRemActionTiming, reply *string) e
 // Returns a list of ActionTriggers on an account
 func (self *ApierV1) GetAccountActionTriggers(attrs AttrAcntAction, reply *engine.ActionTriggerPriotityList) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Direction"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if balance, err := self.AccountDb.GetAccount(utils.BalanceKey(attrs.Tenant, attrs.Account, attrs.Direction)); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if balance, err := self.AccountDb.GetAccount(utils.AccountKey(attrs.Tenant, attrs.Account, attrs.Direction)); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = balance.ActionTriggers
 	}
@@ -117,62 +118,52 @@ func (self *ApierV1) GetAccountActionTriggers(attrs AttrAcntAction, reply *engin
 }
 
 type AttrRemAcntActionTriggers struct {
-	Tenant          string // Tenant he account belongs to
-	Account         string // Account name
-	Direction       string // Traffic direction
-	ActionTriggerId string // Id filtering only specific id to remove
+	Tenant           string // Tenant he account belongs to
+	Account          string // Account name
+	Direction        string // Traffic direction
+	ActionTriggersId string // Id filtering only specific id to remove (can be regexp pattern)
 }
 
 // Returns a list of ActionTriggers on an account
 func (self *ApierV1) RemAccountActionTriggers(attrs AttrRemAcntActionTriggers, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Direction"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	balanceId := utils.BalanceKey(attrs.Tenant, attrs.Account, attrs.Direction)
-	_, err := engine.AccLock.Guard(balanceId, func() (float64, error) {
+	balanceId := utils.AccountKey(attrs.Tenant, attrs.Account, attrs.Direction)
+	_, err := engine.Guardian.Guard(func() (interface{}, error) {
 		ub, err := self.AccountDb.GetAccount(balanceId)
 		if err != nil {
 			return 0, err
 		}
-		for idx, actr := range ub.ActionTriggers {
-			if len(attrs.ActionTriggerId) != 0 && actr.Id != attrs.ActionTriggerId { // Empty actionTriggerId will match always
-				continue
-			}
-			if len(ub.ActionTriggers) != 1 { // Remove by index
-				ub.ActionTriggers[idx], ub.ActionTriggers = ub.ActionTriggers[len(ub.ActionTriggers)-1], ub.ActionTriggers[:len(ub.ActionTriggers)-1]
-			} else { // For last item, simply reinit the slice
-				ub.ActionTriggers = make(engine.ActionTriggerPriotityList, 0)
+		nactrs := make(engine.ActionTriggerPriotityList, 0)
+		for _, actr := range ub.ActionTriggers {
+			match, _ := regexp.MatchString(attrs.ActionTriggersId, actr.Id)
+			if len(attrs.ActionTriggersId) != 0 && !match {
+				nactrs = append(nactrs, actr)
 			}
 		}
+		ub.ActionTriggers = nactrs
 		if err := self.AccountDb.SetAccount(ub); err != nil {
 			return 0, err
 		}
 		return 0, nil
-	})
+	}, balanceId)
 	if err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	}
 	*reply = OK
 	return nil
 }
 
-type AttrSetAccount struct {
-	Tenant        string
-	Direction     string
-	Account       string
-	ActionPlanId  string
-	AllowNegative bool
-}
-
 // Ads a new account into dataDb. If already defined, returns success.
-func (self *ApierV1) SetAccount(attr AttrSetAccount, reply *string) error {
+func (self *ApierV1) SetAccount(attr utils.AttrSetAccount, reply *string) error {
 	if missing := utils.MissingStructFields(&attr, []string{"Tenant", "Direction", "Account"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	balanceId := utils.BalanceKey(attr.Tenant, attr.Account, attr.Direction)
+	balanceId := utils.AccountKey(attr.Tenant, attr.Account, attr.Direction)
 	var ub *engine.Account
-	var ats engine.ActionPlan
-	_, err := engine.AccLock.Guard(balanceId, func() (float64, error) {
+	var ats engine.ActionPlans
+	_, err := engine.Guardian.Guard(func() (interface{}, error) {
 		if bal, _ := self.AccountDb.GetAccount(balanceId); bal != nil {
 			ub = bal
 		} else { // Not found in db, create it here
@@ -184,7 +175,7 @@ func (self *ApierV1) SetAccount(attr AttrSetAccount, reply *string) error {
 
 		if len(attr.ActionPlanId) != 0 {
 			var err error
-			ats, err = self.AccountDb.GetActionTimings(attr.ActionPlanId)
+			ats, err = self.RatingDb.GetActionPlans(attr.ActionPlanId)
 			if err != nil {
 				return 0, err
 			}
@@ -197,25 +188,87 @@ func (self *ApierV1) SetAccount(attr AttrSetAccount, reply *string) error {
 			return 0, err
 		}
 		return 0, nil
-	})
+	}, balanceId)
 	if err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	}
 	if len(ats) != 0 {
-		_, err := engine.AccLock.Guard(engine.ACTION_TIMING_PREFIX, func() (float64, error) { // ToDo: Try locking it above on read somehow
-			if err := self.AccountDb.SetActionTimings(attr.ActionPlanId, ats); err != nil {
+		_, err := engine.Guardian.Guard(func() (interface{}, error) { // ToDo: Try locking it above on read somehow
+			if err := self.RatingDb.SetActionPlans(attr.ActionPlanId, ats); err != nil {
 				return 0, err
 			}
 			return 0, nil
-		})
+		}, utils.ACTION_TIMING_PREFIX)
 		if err != nil {
-			return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+			return utils.NewErrServerError(err)
 		}
 		if self.Sched != nil {
-			self.Sched.LoadActionTimings(self.AccountDb)
+			self.Sched.LoadActionPlans(self.RatingDb)
 			self.Sched.Restart()
 		}
 	}
 	*reply = OK // This will mark saving of the account, error still can show up in actionTimingsId
 	return nil
 }
+
+func (self *ApierV1) RemoveAccount(attr utils.AttrRemoveAccount, reply *string) error {
+	if missing := utils.MissingStructFields(&attr, []string{"Tenant", "Direction", "Account"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	accountId := utils.AccountKey(attr.Tenant, attr.Account, attr.Direction)
+	_, err := engine.Guardian.Guard(func() (interface{}, error) {
+		if err := self.AccountDb.RemoveAccount(accountId); err != nil {
+			return 0, err
+		}
+		return 0, nil
+	}, accountId)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+
+	*reply = OK
+	return nil
+}
+
+func (self *ApierV1) GetAccounts(attr utils.AttrGetAccounts, reply *[]*engine.Account) error {
+	if len(attr.Tenant) == 0 {
+		return utils.NewErrMandatoryIeMissing("Tenanat")
+	}
+	if len(attr.Direction) == 0 {
+		attr.Direction = utils.OUT
+	}
+	var accountKeys []string
+	var err error
+	if len(attr.AccountIds) == 0 {
+		if accountKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACCOUNT_PREFIX + utils.ConcatenatedKey(attr.Direction, attr.Tenant)); err != nil {
+			return err
+		}
+	} else {
+		for _, acntId := range attr.AccountIds {
+			if len(acntId) == 0 { // Source of error returned from redis (key not found)
+				continue
+			}
+			accountKeys = append(accountKeys, utils.ACCOUNT_PREFIX+utils.ConcatenatedKey(attr.Direction, attr.Tenant, acntId))
+		}
+	}
+	if len(accountKeys) == 0 {
+		return nil
+	}
+	var limitedAccounts []string
+	if attr.Limit != 0 {
+		max := math.Min(float64(attr.Offset+attr.Limit), float64(len(accountKeys)))
+		limitedAccounts = accountKeys[attr.Offset:int(max)]
+	} else {
+		limitedAccounts = accountKeys[attr.Offset:]
+	}
+	retAccounts := make([]*engine.Account, 0)
+	for _, acntKey := range limitedAccounts {
+		if acnt, err := self.AccountDb.GetAccount(acntKey[len(utils.ACCOUNT_PREFIX):]); err != nil && err != utils.ErrNotFound { // Not found is not an error here
+			return err
+		} else if acnt != nil {
+			retAccounts = append(retAccounts, acnt)
+		}
+	}
+	*reply = retAccounts
+	return nil
+}

apier/v1/accounts_test.go

@@ -0,0 +1,106 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"testing"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var (
+	apierAcnts            *ApierV1
+	apierAcntsAcntStorage *engine.MapStorage
+)
+
+func init() {
+	apierAcntsAcntStorage, _ = engine.NewMapStorage()
+	cfg, _ := config.NewDefaultCGRConfig()
+	apierAcnts = &ApierV1{AccountDb: engine.AccountingStorage(apierAcntsAcntStorage), Config: cfg}
+}
+
+func TestSetAccounts(t *testing.T) {
+	cgrTenant := "cgrates.org"
+	iscTenant := "itsyscom.com"
+	b10 := &engine.Balance{Value: 10, Weight: 10}
+	cgrAcnt1 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, cgrTenant, "account1"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	cgrAcnt2 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, cgrTenant, "account2"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	cgrAcnt3 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, cgrTenant, "account3"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	iscAcnt1 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, iscTenant, "account1"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	iscAcnt2 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, iscTenant, "account2"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	for _, account := range []*engine.Account{cgrAcnt1, cgrAcnt2, cgrAcnt3, iscAcnt1, iscAcnt2} {
+		if err := apierAcntsAcntStorage.SetAccount(account); err != nil {
+			t.Error(err)
+		}
+	}
+	apierAcntsAcntStorage.CachePrefixes(utils.ACTION_PREFIX)
+}
+
+/*
+func TestGetAccountIds(t *testing.T) {
+	var accountIds []string
+	var attrs AttrGetAccountIds
+	if err := apierAcnts.GetAccountIds(attrs, &accountIds); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accountIds) != 5 {
+		t.Errorf("Accounts returned: %+v", accountIds)
+	}
+}
+*/
+
+func TestGetAccounts(t *testing.T) {
+	var accounts []*engine.Account
+	var attrs utils.AttrGetAccounts
+	if err := apierAcnts.GetAccounts(utils.AttrGetAccounts{Tenant: "cgrates.org"}, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 3 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+	attrs = utils.AttrGetAccounts{Tenant: "itsyscom.com"}
+	if err := apierAcnts.GetAccounts(attrs, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 2 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+	attrs = utils.AttrGetAccounts{Tenant: "cgrates.org", AccountIds: []string{"account1"}}
+	if err := apierAcnts.GetAccounts(attrs, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 1 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+	attrs = utils.AttrGetAccounts{Tenant: "itsyscom.com", AccountIds: []string{"INVALID"}}
+	if err := apierAcnts.GetAccounts(attrs, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 0 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+	attrs = utils.AttrGetAccounts{Tenant: "INVALID"}
+	if err := apierAcnts.GetAccounts(attrs, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 0 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+}

apier/v1/aliases.go

@@ -0,0 +1,140 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type AttrAddRatingSubjectAliases struct {
+	Tenant, Subject string
+	Aliases         []string
+}
+
+type AttrAddAccountAliases struct {
+	Tenant, Account string
+	Aliases         []string
+}
+
+// Retrieve aliases configured for a rating profile subject
+func (self *ApierV1) AddRatingSubjectAliases(attrs AttrAddRatingSubjectAliases, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Subject", "Aliases"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	aliasesChanged := []string{}
+	for _, alias := range attrs.Aliases {
+		if err := self.RatingDb.SetRpAlias(utils.RatingSubjectAliasKey(attrs.Tenant, alias), attrs.Subject); err != nil {
+			return utils.NewErrServerError(err)
+		}
+		aliasesChanged = append(aliasesChanged, utils.RP_ALIAS_PREFIX+utils.RatingSubjectAliasKey(attrs.Tenant, alias))
+	}
+	if err := self.RatingDb.CachePrefixValues(map[string][]string{utils.RP_ALIAS_PREFIX: aliasesChanged}); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
+}
+
+// Retrieve aliases configured for a rating profile subject
+func (self *ApierV1) GetRatingSubjectAliases(attrs engine.TenantRatingSubject, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Subject"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if aliases, err := self.RatingDb.GetRPAliases(attrs.Tenant, attrs.Subject, false); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(aliases) == 0 { // Need it since otherwise we get some unexpected errrors in the client
+		return utils.ErrNotFound
+	} else {
+		*reply = aliases
+	}
+	return nil
+}
+
+// Retrieve aliases configured for a rating profile subject
+func (self *ApierV1) RemRatingSubjectAliases(tenantRatingSubject engine.TenantRatingSubject, reply *string) error {
+	if missing := utils.MissingStructFields(&tenantRatingSubject, []string{"Tenant", "Subject"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.RatingDb.RemoveRpAliases([]*engine.TenantRatingSubject{&tenantRatingSubject}, false); err != nil {
+		if err == utils.ErrNotFound {
+			return err
+		}
+		return utils.NewErrServerError(err)
+	}
+
+	// cache refresh not needed, synched in RemoveRpAliases
+	/*if err := self.RatingDb.CachePrefixes(utils.RP_ALIAS_PREFIX); err != nil {
+		return utils.NewErrServerError(err)
+	}*/
+	*reply = utils.OK
+	return nil
+}
+
+func (self *ApierV1) AddAccountAliases(attrs AttrAddAccountAliases, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Aliases"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	aliasesChanged := []string{}
+	for _, alias := range attrs.Aliases {
+		if err := self.RatingDb.SetAccAlias(utils.AccountAliasKey(attrs.Tenant, alias), attrs.Account); err != nil {
+			return utils.NewErrServerError(err)
+		}
+		aliasesChanged = append(aliasesChanged, utils.ACC_ALIAS_PREFIX+utils.AccountAliasKey(attrs.Tenant, alias))
+	}
+	if err := self.RatingDb.CachePrefixValues(map[string][]string{utils.ACC_ALIAS_PREFIX: aliasesChanged}); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
+}
+
+// Retrieve aliases configured for an account
+func (self *ApierV1) GetAccountAliases(attrs engine.TenantAccount, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if aliases, err := self.RatingDb.GetAccountAliases(attrs.Tenant, attrs.Account, false); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(aliases) == 0 {
+		return utils.ErrNotFound
+	} else {
+		*reply = aliases
+	}
+	return nil
+}
+
+// Retrieve aliases configured for a rating profile subject
+func (self *ApierV1) RemAccountAliases(tenantAccount engine.TenantAccount, reply *string) error {
+	if missing := utils.MissingStructFields(&tenantAccount, []string{"Tenant", "Account"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.RatingDb.RemoveAccAliases([]*engine.TenantAccount{&tenantAccount}, false); err != nil {
+		if err == utils.ErrNotFound {
+			return err
+		}
+		return utils.NewErrServerError(err)
+	}
+	// cache refresh not needed, synched in RemoveRpAliases
+	/*if err := self.RatingDb.CachePrefixes(utils.ACC_ALIAS_PREFIX); err != nil {
+		return utils.NewErrServerError(err)
+	}*/
+	*reply = utils.OK
+	return nil
+}

apier/v1/auth.go

@@ -0,0 +1,74 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Returns MaxUsage (for calls in seconds), -1 for no limit
+func (self *ApierV1) GetMaxUsage(usageRecord engine.UsageRecord, maxUsage *float64) error {
+	out, err := engine.LoadUserProfile(usageRecord, "")
+	if err != nil {
+		return err
+	}
+	usageRecord = out.(engine.UsageRecord)
+	if usageRecord.TOR == "" {
+		usageRecord.TOR = utils.VOICE
+	}
+	if usageRecord.ReqType == "" {
+		usageRecord.ReqType = self.Config.DefaultReqType
+	}
+	if usageRecord.Direction == "" {
+		usageRecord.Direction = utils.OUT
+	}
+	if usageRecord.Tenant == "" {
+		usageRecord.Tenant = self.Config.DefaultTenant
+	}
+	if usageRecord.Category == "" {
+		usageRecord.Category = self.Config.DefaultCategory
+	}
+	if usageRecord.Subject == "" {
+		usageRecord.Subject = usageRecord.Account
+	}
+	if usageRecord.SetupTime == "" {
+		usageRecord.SetupTime = utils.META_NOW
+	}
+	if usageRecord.Usage == "" {
+		usageRecord.Usage = strconv.FormatFloat(self.Config.MaxCallDuration.Seconds(), 'f', -1, 64)
+	}
+	storedCdr, err := usageRecord.AsStoredCdr()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	var maxDur float64
+	if err := self.Responder.GetDerivedMaxSessionTime(storedCdr, &maxDur); err != nil {
+		return err
+	}
+	if maxDur == -1.0 {
+		*maxUsage = -1.0
+		return nil
+	}
+	*maxUsage = time.Duration(maxDur).Seconds()
+	return nil
+}

apier/v1/cdre.go

@@ -0,0 +1,208 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/cgrates/cgrates/cdre"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func (self *ApierV1) ExportCdrsToZipString(attr utils.AttrExpFileCdrs, reply *string) error {
+	tmpDir := "/tmp"
+	attr.ExportDir = &tmpDir // Enforce exporting to tmp always so we avoid cleanup issues
+	efc := utils.ExportedFileCdrs{}
+	if err := self.ExportCdrsToFile(attr, &efc); err != nil {
+		return err
+	} else if efc.TotalRecords == 0 || len(efc.ExportedFilePath) == 0 {
+		return errors.New("No CDR records to export")
+	}
+	// Create a buffer to write our archive to.
+	buf := new(bytes.Buffer)
+	// Create a new zip archive.
+	w := zip.NewWriter(buf)
+	// read generated file
+	content, err := ioutil.ReadFile(efc.ExportedFilePath)
+	if err != nil {
+		return err
+	}
+	exportFileName := path.Base(efc.ExportedFilePath)
+	f, err := w.Create(exportFileName)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(content)
+	if err != nil {
+		return err
+	}
+	// Write metadata into a separate file with extension .cgr
+	medaData, err := json.MarshalIndent(efc, "", "  ")
+	if err != nil {
+		errors.New("Failed creating metadata content")
+	}
+	medatadaFileName := exportFileName[:len(path.Ext(exportFileName))] + ".cgr"
+	mf, err := w.Create(medatadaFileName)
+	if err != nil {
+		return err
+	}
+	_, err = mf.Write(medaData)
+	if err != nil {
+		return err
+	}
+	// Make sure to check the error on Close.
+	if err := w.Close(); err != nil {
+		return err
+	}
+	if err := os.Remove(efc.ExportedFilePath); err != nil {
+		fmt.Errorf("Failed removing exported file at path: %s", efc.ExportedFilePath)
+	}
+	*reply = base64.StdEncoding.EncodeToString(buf.Bytes())
+	return nil
+}
+
+// Export Cdrs to file
+func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.ExportedFileCdrs) error {
+	var err error
+	exportTemplate := self.Config.CdreProfiles[utils.META_DEFAULT]
+	if attr.ExportTemplate != nil && len(*attr.ExportTemplate) != 0 { // Export template prefered, use it
+		var hasIt bool
+		if exportTemplate, hasIt = self.Config.CdreProfiles[*attr.ExportTemplate]; !hasIt {
+			return fmt.Errorf("%s:ExportTemplate", utils.ErrNotFound.Error())
+		}
+	}
+	if exportTemplate == nil {
+		return fmt.Errorf("%s:ExportTemplate", utils.ErrMandatoryIeMissing.Error())
+	}
+	cdrFormat := exportTemplate.CdrFormat
+	if attr.CdrFormat != nil && len(*attr.CdrFormat) != 0 {
+		cdrFormat = strings.ToLower(*attr.CdrFormat)
+	}
+	if !utils.IsSliceMember(utils.CdreCdrFormats, cdrFormat) {
+		return fmt.Errorf("%s:%s", utils.ErrMandatoryIeMissing.Error(), "CdrFormat")
+	}
+	fieldSep := exportTemplate.FieldSeparator
+	if attr.FieldSeparator != nil && len(*attr.FieldSeparator) != 0 {
+		fieldSep, _ = utf8.DecodeRuneInString(*attr.FieldSeparator)
+		if fieldSep == utf8.RuneError {
+			return fmt.Errorf("%s:FieldSeparator:%s", utils.ErrServerError.Error(), "Invalid")
+		}
+	}
+	exportDir := exportTemplate.ExportDir
+	if attr.ExportDir != nil && len(*attr.ExportDir) != 0 {
+		exportDir = *attr.ExportDir
+	}
+	exportId := strconv.FormatInt(time.Now().Unix(), 10)
+	if attr.ExportId != nil && len(*attr.ExportId) != 0 {
+		exportId = *attr.ExportId
+	}
+	fileName := fmt.Sprintf("cdre_%s.%s", exportId, cdrFormat)
+	if attr.ExportFileName != nil && len(*attr.ExportFileName) != 0 {
+		fileName = *attr.ExportFileName
+	}
+	filePath := path.Join(exportDir, fileName)
+	if cdrFormat == utils.DRYRUN {
+		filePath = utils.DRYRUN
+	}
+	dataUsageMultiplyFactor := exportTemplate.DataUsageMultiplyFactor
+	if attr.DataUsageMultiplyFactor != nil && *attr.DataUsageMultiplyFactor != 0.0 {
+		dataUsageMultiplyFactor = *attr.DataUsageMultiplyFactor
+	}
+	smsUsageMultiplyFactor := exportTemplate.SmsUsageMultiplyFactor
+	if attr.SmsUsageMultiplyFactor != nil && *attr.SmsUsageMultiplyFactor != 0.0 {
+		smsUsageMultiplyFactor = *attr.SmsUsageMultiplyFactor
+	}
+	genericUsageMultiplyFactor := exportTemplate.GenericUsageMultiplyFactor
+	if attr.GenericUsageMultiplyFactor != nil && *attr.GenericUsageMultiplyFactor != 0.0 {
+		genericUsageMultiplyFactor = *attr.GenericUsageMultiplyFactor
+	}
+	costMultiplyFactor := exportTemplate.CostMultiplyFactor
+	if attr.CostMultiplyFactor != nil && *attr.CostMultiplyFactor != 0.0 {
+		costMultiplyFactor = *attr.CostMultiplyFactor
+	}
+	costShiftDigits := exportTemplate.CostShiftDigits
+	if attr.CostShiftDigits != nil {
+		costShiftDigits = *attr.CostShiftDigits
+	}
+	roundingDecimals := exportTemplate.CostRoundingDecimals
+	if attr.RoundDecimals != nil {
+		roundingDecimals = *attr.RoundDecimals
+	}
+	maskDestId := exportTemplate.MaskDestId
+	if attr.MaskDestinationId != nil && len(*attr.MaskDestinationId) != 0 {
+		maskDestId = *attr.MaskDestinationId
+	}
+	maskLen := exportTemplate.MaskLength
+	if attr.MaskLength != nil {
+		maskLen = *attr.MaskLength
+	}
+	cdrsFltr, err := attr.AsCdrsFilter()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	cdrs, _, err := self.CdrDb.GetStoredCdrs(cdrsFltr)
+	if err != nil {
+		return err
+	} else if len(cdrs) == 0 {
+		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
+		return nil
+	}
+	cdrexp, err := cdre.NewCdrExporter(cdrs, self.CdrDb, exportTemplate, cdrFormat, fieldSep, exportId, dataUsageMultiplyFactor, smsUsageMultiplyFactor, genericUsageMultiplyFactor,
+		costMultiplyFactor, costShiftDigits, roundingDecimals, self.Config.RoundingDecimals, maskDestId, maskLen, self.Config.HttpSkipTlsVerify)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if cdrexp.TotalExportedCdrs() == 0 {
+		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
+		return nil
+	}
+	if err := cdrexp.WriteToFile(filePath); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), TotalCost: cdrexp.TotalCost(), FirstOrderId: cdrexp.FirstOrderId(), LastOrderId: cdrexp.LastOrderId()}
+	if !attr.SuppressCgrIds {
+		reply.ExportedCgrIds = cdrexp.PositiveExports()
+		reply.UnexportedCgrIds = cdrexp.NegativeExports()
+	}
+	return nil
+}
+
+// Remove Cdrs out of CDR storage
+func (self *ApierV1) RemCdrs(attrs utils.AttrRemCdrs, reply *string) error {
+	if len(attrs.CgrIds) == 0 {
+		return fmt.Errorf("%s:CgrIds", utils.ErrMandatoryIeMissing.Error())
+	}
+	if err := self.CdrDb.RemStoredCdrs(attrs.CgrIds); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}

apier/v1/cdrs.go

@@ -0,0 +1,60 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Retrieves the callCost out of CGR logDb
+func (apier *ApierV1) GetCallCostLog(attrs utils.AttrGetCallCost, reply *engine.CallCost) error {
+	if attrs.CgrId == "" {
+		return utils.NewErrMandatoryIeMissing("CgrId")
+	}
+	if attrs.RunId == "" {
+		attrs.RunId = utils.META_DEFAULT
+	}
+	if cc, err := apier.CdrDb.GetCallCostLog(attrs.CgrId, "", attrs.RunId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if cc == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = *cc
+	}
+	return nil
+}
+
+// Retrieves CDRs based on the filters
+func (apier *ApierV1) GetCdrs(attrs utils.AttrGetCdrs, reply *[]*engine.ExternalCdr) error {
+	cdrsFltr, err := attrs.AsCdrsFilter()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if cdrs, _, err := apier.CdrDb.GetStoredCdrs(cdrsFltr); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(cdrs) == 0 {
+		*reply = make([]*engine.ExternalCdr, 0)
+	} else {
+		for _, cdr := range cdrs {
+			*reply = append(*reply, cdr.AsExternalCdr())
+		}
+	}
+	return nil
+}

apier/v1/cdrstatsv1.go

@@ -0,0 +1,70 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"fmt"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Interact with Stats server
+type CDRStatsV1 struct {
+	CdrStats engine.StatsInterface
+}
+
+type AttrGetMetrics struct {
+	StatsQueueId string // Id of the stats instance queried
+}
+
+func (sts *CDRStatsV1) GetMetrics(attr AttrGetMetrics, reply *map[string]float64) error {
+	if len(attr.StatsQueueId) == 0 {
+		return fmt.Errorf("%s:StatsQueueId", utils.ErrMandatoryIeMissing.Error())
+	}
+	return sts.CdrStats.GetValues(attr.StatsQueueId, reply)
+}
+
+func (sts *CDRStatsV1) GetQueueIds(empty string, reply *[]string) error {
+	return sts.CdrStats.GetQueueIds(0, reply)
+}
+
+func (sts *CDRStatsV1) GetQueue(id string, sq *engine.StatsQueue) error {
+	return sts.CdrStats.GetQueue(id, sq)
+}
+
+func (sts *CDRStatsV1) GetQueueTriggers(id string, ats *engine.ActionTriggerPriotityList) error {
+	return sts.CdrStats.GetQueueTriggers(id, ats)
+}
+
+func (sts *CDRStatsV1) ReloadQueues(attr utils.AttrCDRStatsReloadQueues, reply *string) error {
+	if err := sts.CdrStats.ReloadQueues(attr.StatsQueueIds, nil); err != nil {
+		return err
+	}
+	*reply = utils.OK
+	return nil
+}
+
+func (sts *CDRStatsV1) ResetQueues(attr utils.AttrCDRStatsReloadQueues, reply *string) error {
+	if err := sts.CdrStats.ResetQueues(attr.StatsQueueIds, nil); err != nil {
+		return err
+	}
+	*reply = utils.OK
+	return nil
+}

apier/v1/cdrstatsv1_local_test.go

@@ -0,0 +1,204 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"fmt"
+	"net/http"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"path"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var cdrstCfgPath string
+var cdrstCfg *config.CGRConfig
+var cdrstRpc *rpc.Client
+
+func TestCDRStatsLclLoadConfig(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrstCfgPath = path.Join(*dataDir, "conf", "samples", "cdrstats")
+	if cdrstCfg, err = config.NewCGRConfigFromFolder(cfgPath); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestCDRStatsLclInitDataDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitDataDb(cdrstCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCDRStatsLclStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if _, err := engine.StopStartEngine(cdrstCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestCDRStatsLclRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrstRpc, err = jsonrpc.Dial("tcp", cdrstCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+func TestCDRStatsLclLoadTariffPlanFromFolder(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	reply := ""
+	// Simple test that command is executed without errors
+	attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tariffplans", "cdrstats")}
+	if err := cdrstRpc.Call("ApierV1.LoadTariffPlanFromFolder", attrs, &reply); err != nil {
+		t.Error("Got error on ApierV1.LoadTariffPlanFromFolder: ", err.Error())
+	} else if reply != "OK" {
+		t.Error("Calling ApierV1.LoadTariffPlanFromFolder got reply: ", reply)
+	}
+	time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups
+}
+
+func TestCDRStatsLclGetQueueIds2(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var queueIds []string
+	eQueueIds := []string{"CDRST3", "CDRST4"}
+	if err := cdrstRpc.Call("CDRStatsV1.GetQueueIds", "", &queueIds); err != nil {
+		t.Error("Calling CDRStatsV1.GetQueueIds, got error: ", err.Error())
+	} else if len(eQueueIds) != len(queueIds) {
+		t.Errorf("Expecting: %v, received: %v", eQueueIds, queueIds)
+	}
+}
+
+func TestCDRStatsLclPostCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	httpClient := new(http.Client)
+	storedCdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsafa", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(),
+			AnswerTime: time.Now(), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
+			Cost: 1.01, RatedAccount: "dan", RatedSubject: "dan",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsafb", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(),
+			AnswerTime: time.Now(), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(5) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dan",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsafc", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), AnswerTime: time.Now(),
+			MediationRunId: utils.DEFAULT_RUNID,
+			Usage:          time.Duration(30) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dan",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsafd", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), AnswerTime: time.Time{},
+			MediationRunId: utils.DEFAULT_RUNID,
+			Usage:          time.Duration(0) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dan",
+		},
+	}
+	for _, storedCdr := range storedCdrs {
+		if _, err := httpClient.PostForm(fmt.Sprintf("http://%s/cdr_http", "127.0.0.1:2080"), storedCdr.AsHttpForm()); err != nil {
+			t.Error(err.Error())
+		}
+	}
+	time.Sleep(time.Duration(*waitRater) * time.Millisecond)
+
+}
+
+func TestCDRStatsLclGetMetrics1(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var rcvMetrics2 map[string]float64
+	expectedMetrics2 := map[string]float64{"ASR": 75, "ACD": 15}
+	if err := cdrstRpc.Call("CDRStatsV1.GetMetrics", AttrGetMetrics{StatsQueueId: "CDRST4"}, &rcvMetrics2); err != nil {
+		t.Error("Calling CDRStatsV1.GetMetrics, got error: ", err.Error())
+	} else if !reflect.DeepEqual(expectedMetrics2, rcvMetrics2) {
+		t.Errorf("Expecting: %v, received: %v", expectedMetrics2, rcvMetrics2)
+	}
+}
+
+// Test stats persistence
+func TestCDRStatsLclStatsPersistence(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	time.Sleep(time.Duration(2) * time.Second) // Allow stats to be updated in dataDb
+	if _, err := engine.StopStartEngine(cdrstCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+	var err error
+	cdrstRpc, err = jsonrpc.Dial("tcp", cdrstCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+	var rcvMetrics map[string]float64
+	expectedMetrics := map[string]float64{"ASR": 75, "ACD": 15}
+	if err := cdrstRpc.Call("CDRStatsV1.GetMetrics", AttrGetMetrics{StatsQueueId: "CDRST4"}, &rcvMetrics); err != nil {
+		t.Error("Calling CDRStatsV1.GetMetrics, got error: ", err.Error())
+	} else if !reflect.DeepEqual(expectedMetrics, rcvMetrics) {
+		t.Errorf("Expecting: %v, received: %v", expectedMetrics, rcvMetrics)
+	}
+}
+
+func TestCDRStatsLclResetMetrics(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	if err := cdrstRpc.Call("CDRStatsV1.ResetQueues", utils.AttrCDRStatsReloadQueues{StatsQueueIds: []string{"CDRST4"}}, &reply); err != nil {
+		t.Error("Calling CDRStatsV1.ResetQueues, got error: ", err.Error())
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply received: ", reply)
+	}
+	time.Sleep(time.Duration(*waitRater) * time.Millisecond)
+	var rcvMetrics2 map[string]float64
+	expectedMetrics2 := map[string]float64{"ASR": -1, "ACD": -1}
+	if err := cdrstRpc.Call("CDRStatsV1.GetMetrics", AttrGetMetrics{StatsQueueId: "CDRST4"}, &rcvMetrics2); err != nil {
+		t.Error("Calling CDRStatsV1.GetMetrics, got error: ", err.Error())
+	} else if !reflect.DeepEqual(expectedMetrics2, rcvMetrics2) {
+		t.Errorf("Expecting: %v, received: %v", expectedMetrics2, rcvMetrics2)
+	}
+}

apier/v1/cdrsv1.go

@@ -0,0 +1,74 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"time"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Receive CDRs via RPC methods
+type CdrsV1 struct {
+	CdrSrv *engine.CdrServer
+}
+
+// Designed for CGR internal usage
+func (self *CdrsV1) ProcessCdr(cdr *engine.StoredCdr, reply *string) error {
+	if err := self.CdrSrv.ProcessCdr(cdr); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
+}
+
+// Designed for external programs feeding CDRs to CGRateS
+func (self *CdrsV1) ProcessExternalCdr(cdr *engine.ExternalCdr, reply *string) error {
+	if err := self.CdrSrv.ProcessExternalCdr(cdr); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
+}
+
+// Remotely start mediation with specific runid, runs asynchronously, it's status will be displayed in syslog
+func (self *CdrsV1) RateCdrs(attrs utils.AttrRateCdrs, reply *string) error {
+	var tStart, tEnd time.Time
+	var err error
+	if len(attrs.TimeStart) != 0 {
+		if tStart, err = utils.ParseTimeDetectLayout(attrs.TimeStart); err != nil {
+			return err
+		}
+	}
+	if len(attrs.TimeEnd) != 0 {
+		if tEnd, err = utils.ParseTimeDetectLayout(attrs.TimeEnd); err != nil {
+			return err
+		}
+	}
+	//RateCdrs(cgrIds, runIds, tors, cdrHosts, cdrSources, reqTypes, directions, tenants, categories, accounts, subjects, destPrefixes []string,
+	//orderIdStart, orderIdEnd int64, timeStart, timeEnd time.Time, rerateErrors, rerateRated bool)
+	if err := self.CdrSrv.RateCdrs(attrs.CgrIds, attrs.MediationRunIds, attrs.TORs, attrs.CdrHosts, attrs.CdrSources, attrs.ReqTypes, attrs.Directions,
+		attrs.Tenants, attrs.Categories, attrs.Accounts, attrs.Subjects, attrs.DestinationPrefixes, attrs.RatedAccounts, attrs.RatedSubjects,
+		attrs.OrderIdStart, attrs.OrderIdEnd, tStart, tEnd, attrs.RerateErrors, attrs.RerateRated, attrs.SendToStats); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
+}

apier/v1/costs.go

@@ -0,0 +1,59 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"time"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type AttrGetDataCost struct {
+	Direction                string
+	Category                 string
+	Tenant, Account, Subject string
+	StartTime                time.Time
+	Usage                    int64 // the call duration so far (till TimeEnd)
+}
+
+func (apier *ApierV1) GetDataCost(attrs AttrGetDataCost, reply *engine.DataCost) error {
+	usageAsDuration := time.Duration(attrs.Usage) * time.Second // Convert to seconds to match the loaded rates
+	cd := &engine.CallDescriptor{
+		Direction:     attrs.Direction,
+		Category:      attrs.Category,
+		Tenant:        attrs.Tenant,
+		Account:       attrs.Account,
+		Subject:       attrs.Subject,
+		TimeStart:     attrs.StartTime,
+		TimeEnd:       attrs.StartTime.Add(usageAsDuration),
+		DurationIndex: usageAsDuration,
+		TOR:           utils.DATA,
+	}
+	var cc engine.CallCost
+	if err := apier.Responder.GetCost(cd, &cc); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if dc, err := cc.ToDataCost(); err != nil {
+		return utils.NewErrServerError(err)
+	} else if dc != nil {
+		*reply = *dc
+	}
+	return nil
+}

apier/v1/debit.go

@@ -0,0 +1,67 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func (self *ApierV1) DebitUsage(usageRecord engine.UsageRecord, reply *string) error {
+	if missing := utils.MissingStructFields(&usageRecord, []string{"Account", "Destination", "Usage"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	out, err := engine.LoadUserProfile(usageRecord, "")
+	if err != nil {
+		*reply = err.Error()
+		return err
+	}
+	usageRecord = out.(engine.UsageRecord)
+	if usageRecord.TOR == "" {
+		usageRecord.TOR = utils.VOICE
+	}
+	if usageRecord.ReqType == "" {
+		usageRecord.ReqType = self.Config.DefaultReqType
+	}
+	if usageRecord.Direction == "" {
+		usageRecord.Direction = utils.OUT
+	}
+	if usageRecord.Tenant == "" {
+		usageRecord.Tenant = self.Config.DefaultTenant
+	}
+	if usageRecord.Category == "" {
+		usageRecord.Category = self.Config.DefaultCategory
+	}
+	if usageRecord.Subject == "" {
+		usageRecord.Subject = usageRecord.Account
+	}
+	if usageRecord.AnswerTime == "" {
+		usageRecord.AnswerTime = utils.META_NOW
+	}
+	cd, err := usageRecord.AsCallDescriptor()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	var cc engine.CallCost
+	if err := self.Responder.Debit(cd, &cc); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = OK
+	return nil
+}

apier/v1/derivedcharging.go

@@ -0,0 +1,117 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"fmt"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Get DerivedChargers applying to our call, appends general configured to account specific ones if that is configured
+func (self *ApierV1) GetDerivedChargers(attrs utils.AttrDerivedChargers, reply *utils.DerivedChargers) (err error) {
+	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Direction", "Account", "Subject"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if hDc, err := engine.HandleGetDerivedChargers(self.RatingDb, &attrs); err != nil {
+		return utils.NewErrServerError(err)
+	} else if hDc != nil {
+		*reply = hDc
+	}
+	return nil
+}
+
+type AttrSetDerivedChargers struct {
+	Direction, Tenant, Category, Account, Subject string
+	DerivedChargers                               utils.DerivedChargers
+	Overwrite                                     bool // Do not overwrite if present in redis
+}
+
+func (self *ApierV1) SetDerivedChargers(attrs AttrSetDerivedChargers, reply *string) (err error) {
+	if len(attrs.Direction) == 0 {
+		attrs.Direction = utils.OUT
+	}
+	if len(attrs.Tenant) == 0 {
+		attrs.Tenant = utils.ANY
+	}
+	if len(attrs.Category) == 0 {
+		attrs.Category = utils.ANY
+	}
+	if len(attrs.Account) == 0 {
+		attrs.Account = utils.ANY
+	}
+	if len(attrs.Subject) == 0 {
+		attrs.Subject = utils.ANY
+	}
+	for _, dc := range attrs.DerivedChargers {
+		if _, err = utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP); err != nil { // Make sure rules are OK before loading in db
+			return fmt.Errorf("%s:%s", utils.ErrParserError.Error(), err.Error())
+		}
+	}
+	dcKey := utils.DerivedChargersKey(attrs.Direction, attrs.Tenant, attrs.Category, attrs.Account, attrs.Subject)
+	if !attrs.Overwrite {
+		if exists, err := self.RatingDb.HasData(utils.DESTINATION_PREFIX, dcKey); err != nil {
+			return utils.NewErrServerError(err)
+		} else if exists {
+			return utils.ErrExists
+		}
+	}
+	if err := self.RatingDb.SetDerivedChargers(dcKey, attrs.DerivedChargers); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if err := self.RatingDb.CachePrefixValues(map[string][]string{
+		utils.DERIVEDCHARGERS_PREFIX: []string{utils.DERIVEDCHARGERS_PREFIX + dcKey},
+	}); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
+}
+
+type AttrRemDerivedChargers struct {
+	Direction, Tenant, Category, Account, Subject string
+}
+
+func (self *ApierV1) RemDerivedChargers(attrs AttrRemDerivedChargers, reply *string) error {
+	if len(attrs.Direction) == 0 {
+		attrs.Direction = utils.OUT
+	}
+	if len(attrs.Tenant) == 0 {
+		attrs.Tenant = utils.ANY
+	}
+	if len(attrs.Category) == 0 {
+		attrs.Category = utils.ANY
+	}
+	if len(attrs.Account) == 0 {
+		attrs.Account = utils.ANY
+	}
+	if len(attrs.Subject) == 0 {
+		attrs.Subject = utils.ANY
+	}
+	if err := self.RatingDb.SetDerivedChargers(utils.DerivedChargersKey(attrs.Direction, attrs.Tenant, attrs.Category, attrs.Account, attrs.Subject), nil); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	if err := self.RatingDb.CachePrefixes(utils.DERIVEDCHARGERS_PREFIX); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	return nil
+}

apier/v1/derivedcharging_test.go

@@ -0,0 +1,105 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var apierDcT *ApierV1
+
+func init() {
+	dataStorage, _ := engine.NewMapStorage()
+	cfg, _ := config.NewDefaultCGRConfig()
+	apierDcT = &ApierV1{RatingDb: engine.RatingStorage(dataStorage), Config: cfg}
+}
+
+/*
+func TestGetEmptyDC(t *testing.T) {
+	attrs := utils.AttrDerivedChargers{Tenant: "cgrates.org", Category: "call", Direction: "*out", Account: "dan", Subject: "dan"}
+	var dcs utils.DerivedChargers
+	if err := apierDcT.GetDerivedChargers(attrs, &dcs); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if !reflect.DeepEqual(dcs, apierDcT.Config.DerivedChargers) {
+		t.Error("Returned DerivedChargers not matching the configured ones")
+	}
+}
+*/
+
+func TestSetDC(t *testing.T) {
+	dcs1 := utils.DerivedChargers{
+		&utils.DerivedCharger{RunId: "extra1", ReqTypeField: "^prepaid", DirectionField: "*default", TenantField: "*default", CategoryField: "*default",
+			AccountField: "rif", SubjectField: "rif", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"},
+		&utils.DerivedCharger{RunId: "extra2", ReqTypeField: "*default", DirectionField: "*default", TenantField: "*default", CategoryField: "*default",
+			AccountField: "ivo", SubjectField: "ivo", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"},
+	}
+	attrs := AttrSetDerivedChargers{Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "dan", Subject: "dan", DerivedChargers: dcs1}
+	var reply string
+	if err := apierDcT.SetDerivedChargers(attrs, &reply); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply returned", reply)
+	}
+}
+
+func TestGetDC(t *testing.T) {
+	attrs := utils.AttrDerivedChargers{Tenant: "cgrates.org", Category: "call", Direction: "*out", Account: "dan", Subject: "dan"}
+	eDcs := utils.DerivedChargers{
+		&utils.DerivedCharger{RunId: "extra1", ReqTypeField: "^prepaid", DirectionField: "*default", TenantField: "*default", CategoryField: "*default",
+			AccountField: "rif", SubjectField: "rif", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"},
+		&utils.DerivedCharger{RunId: "extra2", ReqTypeField: "*default", DirectionField: "*default", TenantField: "*default", CategoryField: "*default",
+			AccountField: "ivo", SubjectField: "ivo", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"},
+	}
+	var dcs utils.DerivedChargers
+	if err := apierDcT.GetDerivedChargers(attrs, &dcs); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if !reflect.DeepEqual(dcs, eDcs) {
+		t.Errorf("Expecting: %v, received: %v", eDcs, dcs)
+	}
+}
+
+func TestRemDC(t *testing.T) {
+	attrs := AttrRemDerivedChargers{Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "dan", Subject: "dan"}
+	var reply string
+	if err := apierDcT.RemDerivedChargers(attrs, &reply); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply returned", reply)
+	}
+}
+
+/*
+func TestGetEmptyDC2(t *testing.T) {
+	attrs := utils.AttrDerivedChargers{Tenant: "cgrates.org", Category: "call", Direction: "*out", Account: "dan", Subject: "dan"}
+	var dcs utils.DerivedChargers
+	if err := apierDcT.GetDerivedChargers(attrs, &dcs); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if !reflect.DeepEqual(dcs, apierDcT.Config.DerivedChargers) {
+		for _, dc := range dcs {
+			fmt.Printf("Got dc: %v\n", dc)
+		}
+		t.Error("Returned DerivedChargers not matching the configured ones")
+	}
+}
+*/

apier/v1/lcr.go

@@ -0,0 +1,78 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"fmt"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Computes the LCR for a specific request emulating a call
+func (self *ApierV1) GetLcr(lcrReq engine.LcrRequest, lcrReply *engine.LcrReply) error {
+	cd, err := lcrReq.AsCallDescriptor()
+	if err != nil {
+		return err
+	}
+	var lcrQried engine.LCRCost
+	if err := self.Responder.GetLCR(cd, &lcrQried); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if lcrQried.Entry == nil {
+		return utils.ErrNotFound
+	}
+	if lcrQried.HasErrors() {
+		lcrQried.LogErrors()
+		return fmt.Errorf("%s:%s", utils.ErrServerError.Error(), "LCR_COMPUTE_ERRORS")
+	}
+	lcrReply.DestinationId = lcrQried.Entry.DestinationId
+	lcrReply.RPCategory = lcrQried.Entry.RPCategory
+	lcrReply.Strategy = lcrQried.Entry.Strategy
+	for _, qriedSuppl := range lcrQried.SupplierCosts {
+		if dtcs, err := utils.NewDTCSFromRPKey(qriedSuppl.Supplier); err != nil {
+			return utils.NewErrServerError(err)
+		} else {
+			lcrReply.Suppliers = append(lcrReply.Suppliers, &engine.LcrSupplier{Supplier: dtcs.Subject, Cost: qriedSuppl.Cost, QOS: qriedSuppl.QOS})
+		}
+	}
+	return nil
+}
+
+// Computes the LCR for a specific request emulating a call, returns a comma separated list of suppliers
+func (self *ApierV1) GetLcrSuppliers(lcrReq engine.LcrRequest, suppliers *string) (err error) {
+	cd, err := lcrReq.AsCallDescriptor()
+	if err != nil {
+		return err
+	}
+	var lcrQried engine.LCRCost
+	if err := self.Responder.GetLCR(cd, &lcrQried); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if lcrQried.HasErrors() {
+		lcrQried.LogErrors()
+		return fmt.Errorf("%s:%s", utils.ErrServerError.Error(), "LCR_ERRORS")
+	}
+	if suppliersStr, err := lcrQried.SuppliersString(); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*suppliers = suppliersStr
+	}
+	return nil
+}

apier/v1/mediator.go

@@ -0,0 +1,19 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1

apier/v1/scheduler.go

@@ -0,0 +1,171 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"errors"
+	"strings"
+	"time"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+/*
+[
+    {
+        u'ActionsId': u'BONUS_1',
+        u'Uuid': u'5b5ba53b40b1d44380cce52379ec5c0d',
+        u'Weight': 10,
+        u'Timing': {
+            u'Timing': {
+                u'MonthDays': [
+
+                ],
+                u'Months': [
+
+                ],
+                u'WeekDays': [
+
+                ],
+                u'Years': [
+                    2013
+                ],
+                u'StartTime': u'11: 00: 00',
+                u'EndTime': u''
+            },
+            u'Rating': None,
+            u'Weight': 0
+        },
+        u'AccountIds': [
+            u'*out: cgrates.org: 1001',
+            u'*out: cgrates.org: 1002',
+            u'*out: cgrates.org: 1003',
+            u'*out: cgrates.org: 1004',
+            u'*out: cgrates.org: 1005'
+        ],
+        u'Id': u'PREPAID_10'
+    },
+    {
+        u'ActionsId': u'PREPAID_10',
+        u'Uuid': u'b16ab12740e2e6c380ff7660e8b55528',
+        u'Weight': 10,
+        u'Timing': {
+            u'Timing': {
+                u'MonthDays': [
+
+                ],
+                u'Months': [
+
+                ],
+                u'WeekDays': [
+
+                ],
+                u'Years': [
+                    2013
+                ],
+                u'StartTime': u'11: 00: 00',
+                u'EndTime': u''
+            },
+            u'Rating': None,
+            u'Weight': 0
+        },
+        u'AccountIds': [
+            u'*out: cgrates.org: 1001',
+            u'*out: cgrates.org: 1002',
+            u'*out: cgrates.org: 1003',
+            u'*out: cgrates.org: 1004',
+            u'*out: cgrates.org: 1005'
+        ],
+        u'Id': u'PREPAID_10'
+    }
+]
+*/
+
+type AttrsGetScheduledActions struct {
+	Direction, Tenant, Account string
+	TimeStart, TimeEnd         time.Time // Filter based on next runTime
+	utils.Paginator
+}
+
+type ScheduledActions struct {
+	NextRunTime                             time.Time
+	Accounts                                []*utils.DirectionTenantAccount
+	ActionsId, ActionPlanId, ActionPlanUuid string
+}
+
+func (self *ApierV1) GetScheduledActions(attrs AttrsGetScheduledActions, reply *[]*ScheduledActions) error {
+	schedActions := make([]*ScheduledActions, 0)
+	if self.Sched == nil {
+		return errors.New("SCHEDULER_NOT_ENABLED")
+	}
+	scheduledActions := self.Sched.GetQueue()
+	var min, max int
+	if attrs.Paginator.Offset != nil {
+		min = *attrs.Paginator.Offset
+	}
+	if attrs.Paginator.Limit != nil {
+		max = *attrs.Paginator.Limit
+	}
+	if max > len(scheduledActions) {
+		max = len(scheduledActions)
+	}
+	scheduledActions = scheduledActions[min : min+max]
+	for _, qActions := range scheduledActions {
+		sas := &ScheduledActions{ActionsId: qActions.ActionsId, ActionPlanId: qActions.Id, ActionPlanUuid: qActions.Uuid}
+		if attrs.SearchTerm != "" &&
+			!(strings.Contains(sas.ActionPlanId, attrs.SearchTerm) ||
+				strings.Contains(sas.ActionsId, attrs.SearchTerm)) {
+			continue
+		}
+		sas.NextRunTime = qActions.GetNextStartTime(time.Now())
+		if !attrs.TimeStart.IsZero() && sas.NextRunTime.Before(attrs.TimeStart) {
+			continue // Filter here only requests in the filtered interval
+		}
+		if !attrs.TimeEnd.IsZero() && (sas.NextRunTime.After(attrs.TimeEnd) || sas.NextRunTime.Equal(attrs.TimeEnd)) {
+			continue
+		}
+		acntFiltersMatch := false
+		for _, acntKey := range qActions.AccountIds {
+			directionMatched := len(attrs.Direction) == 0
+			tenantMatched := len(attrs.Tenant) == 0
+			accountMatched := len(attrs.Account) == 0
+			dta, _ := utils.NewDTAFromAccountKey(acntKey)
+			sas.Accounts = append(sas.Accounts, dta)
+			// One member matching
+			if !directionMatched && attrs.Direction == dta.Direction {
+				directionMatched = true
+			}
+			if !tenantMatched && attrs.Tenant == dta.Tenant {
+				tenantMatched = true
+			}
+			if !accountMatched && attrs.Account == dta.Account {
+				accountMatched = true
+			}
+			if directionMatched && tenantMatched && accountMatched {
+				acntFiltersMatch = true
+			}
+		}
+		if !acntFiltersMatch {
+			continue
+		}
+		schedActions = append(schedActions, sas)
+	}
+	*reply = schedActions
+	return nil
+}

apier/v1/smv1.go

@@ -0,0 +1,47 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/sessionmanager"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Interact with SessionManager
+type SessionManagerV1 struct {
+	SMs []sessionmanager.SessionManager // List of session managers since we support having more than one active session manager running on one host
+}
+
+func (self *SessionManagerV1) ActiveSessionMangers(ignored string, reply *[]sessionmanager.SessionManager) error {
+	if len(self.SMs) == 0 {
+		return utils.ErrNotFound
+	}
+	*reply = self.SMs
+	return nil
+}
+
+func (self *SessionManagerV1) ActiveSessions(attrs utils.AttrGetSMASessions, reply *[]*sessionmanager.ActiveSession) error {
+	if attrs.SessionManagerIndex > len(self.SMs)-1 {
+		return utils.ErrNotFound
+	}
+	for _, session := range self.SMs[attrs.SessionManagerIndex].Sessions() {
+		*reply = append(*reply, session.AsActiveSessions()...)
+	}
+	return nil
+}

apier/v1/tp.go

@@ -0,0 +1,99 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+// Tariff plan related APIs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type AttrGetTPIds struct {
+}
+
+// Queries tarrif plan identities gathered from all tables.
+func (self *ApierV1) GetTPIds(attrs AttrGetTPIds, reply *[]string) error {
+	if ids, err := self.StorDb.GetTpIds(); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+type AttrImportTPZipFile struct {
+	TPid string
+	File []byte
+}
+
+func (self *ApierV1) ImportTPZipFile(attrs AttrImportTPZipFile, reply *string) error {
+	tmpDir, err := ioutil.TempDir("/tmp", "cgr_")
+	if err != nil {
+		*reply = "ERROR: creating temp directory!"
+		return err
+	}
+	zipFile := filepath.Join(tmpDir, "/file.zip")
+	if err = ioutil.WriteFile(zipFile, attrs.File, os.ModePerm); err != nil {
+		*reply = "ERROR: writing zip file!"
+		return err
+	}
+	if err = utils.Unzip(zipFile, tmpDir); err != nil {
+		*reply = "ERROR: unziping file!"
+		return err
+	}
+	csvfilesFound := false
+	if err = filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error {
+		if !info.IsDir() {
+			return nil
+		}
+		csvFiles, err := filepath.Glob(filepath.Join(path, "*csv"))
+		if csvFiles != nil {
+			if attrs.TPid == "" {
+				*reply = "ERROR: missing TPid!"
+				return err
+			}
+			csvImporter := engine.TPCSVImporter{
+				TPid:     attrs.TPid,
+				StorDb:   self.StorDb,
+				DirPath:  path,
+				Sep:      ',',
+				Verbose:  false,
+				ImportId: "",
+			}
+			if errImport := csvImporter.Run(); errImport != nil {
+				return errImport
+			}
+			csvfilesFound = true
+		}
+		return err
+	}); err != nil || !csvfilesFound {
+		*reply = "ERROR: finding csv files!"
+		return err
+	}
+	os.RemoveAll(tmpDir)
+	*reply = "OK"
+	return nil
+}

apier/v1/tpaccountactions.go

@@ -0,0 +1,166 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new AccountActions profile within a tariff plan
+func (self *ApierV1) SetTPAccountActions(attrs utils.TPAccountActions, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs,
+		[]string{"TPid", "LoadId", "Tenant", "Account", "Direction", "ActionPlanId", "ActionTriggersId"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	aas := engine.APItoModelAccountAction(&attrs)
+	if err := self.StorDb.SetTpAccountActions([]engine.TpAccountAction{*aas}); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPAccountActionsByLoadId struct {
+	TPid   string // Tariff plan id
+	LoadId string // AccountActions id
+}
+
+// Queries specific AccountActions profile on tariff plan
+func (self *ApierV1) GetTPAccountActionsByLoadId(attrs utils.TPAccountActions, reply *[]*utils.TPAccountActions) error {
+	mndtryFlds := []string{"TPid", "LoadId"}
+	if len(attrs.Account) != 0 { // If account provided as filter, make all related fields mandatory
+		mndtryFlds = append(mndtryFlds, "Tenant", "Account", "Direction")
+	}
+	if missing := utils.MissingStructFields(&attrs, mndtryFlds); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	aas := engine.APItoModelAccountAction(&attrs)
+	if aa, err := self.StorDb.GetTpAccountActions(aas); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(aa) == 0 {
+		return utils.ErrNotFound
+	} else {
+
+		tpAa, err := engine.TpAccountActions(aa).GetAccountActions()
+		if err != nil {
+			return err
+		}
+		var acts []*utils.TPAccountActions
+		if len(attrs.Account) != 0 {
+			acts = []*utils.TPAccountActions{tpAa[attrs.KeyId()]}
+		} else {
+			for _, actLst := range tpAa {
+				acts = append(acts, actLst)
+			}
+		}
+		*reply = acts
+	}
+	return nil
+}
+
+type AttrGetTPAccountActions struct {
+	TPid             string // Tariff plan id
+	AccountActionsId string // DerivedCharge id
+}
+
+// Queries specific DerivedCharge on tariff plan
+func (self *ApierV1) GetTPAccountActions(attrs AttrGetTPAccountActions, reply *utils.TPAccountActions) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "AccountActionsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpAa := &utils.TPAccountActions{TPid: attrs.TPid}
+	if err := tmpAa.SetAccountActionsId(attrs.AccountActionsId); err != nil {
+		return err
+	}
+	tmpAaa := engine.APItoModelAccountAction(tmpAa)
+	if aas, err := self.StorDb.GetTpAccountActions(tmpAaa); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(aas) == 0 {
+		return utils.ErrNotFound
+	} else {
+		tpAaa, err := engine.TpAccountActions(aas).GetAccountActions()
+		if err != nil {
+			return err
+		}
+		aa := tpAaa[tmpAa.KeyId()]
+		tpdc := utils.TPAccountActions{
+			TPid:             attrs.TPid,
+			ActionPlanId:     aa.ActionPlanId,
+			ActionTriggersId: aa.ActionTriggersId,
+		}
+		if err := tpdc.SetAccountActionsId(attrs.AccountActionsId); err != nil {
+			return err
+		}
+		*reply = tpdc
+	}
+	return nil
+}
+
+type AttrGetTPAccountActionIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries AccountActions identities on specific tariff plan.
+func (self *ApierV1) GetTPAccountActionLoadIds(attrs AttrGetTPAccountActionIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACCOUNT_ACTIONS, utils.TPDistinctIds{"loadid"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Queries DerivedCharges identities on specific tariff plan.
+func (self *ApierV1) GetTPAccountActionIds(attrs AttrGetTPAccountActionIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACCOUNT_ACTIONS, utils.TPDistinctIds{"loadid", "direction", "tenant", "account"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific AccountActions on Tariff plan
+func (self *ApierV1) RemTPAccountActions(attrs AttrGetTPAccountActions, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "Account", "Direction"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	aa := engine.TpAccountAction{Tpid: attrs.TPid}
+	if err := aa.SetAccountActionId(attrs.AccountActionsId); err != nil {
+		return err
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.Tpid, aa.Loadid, aa.Direction, aa.Tenant, aa.Account); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpactionplans.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,28 +16,29 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"errors"
 	"fmt"
 
+	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
 // Creates a new ActionTimings profile within a tariff plan
 func (self *ApierV1) SetTPActionPlan(attrs utils.TPActionPlan, reply *string) error {
-	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Id", "ActionPlan"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionPlanId", "ActionPlan"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	for _, at := range attrs.ActionPlan {
 		requiredFields := []string{"ActionsId", "TimingId", "Weight"}
 		if missing := utils.MissingStructFields(at, requiredFields); len(missing) != 0 {
-			return fmt.Errorf("%s:Action:%s:%v", utils.ERR_MANDATORY_IE_MISSING, at.ActionsId, missing)
+			return fmt.Errorf("%s:Action:%s:%v", utils.ErrMandatoryIeMissing.Error(), at.ActionsId, missing)
 		}
 	}
-	if err := self.StorDb.SetTPActionTimings(attrs.TPid, map[string][]*utils.TPActionTiming{attrs.Id: attrs.ActionPlan}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	ap := engine.APItoModelActionPlan(&attrs)
+	if err := self.StorDb.SetTpActionPlans(ap); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -45,20 +46,28 @@ func (self *ApierV1) SetTPActionPlan(attrs utils.TPActionPlan, reply *string) er
 
 type AttrGetTPActionPlan struct {
 	TPid string // Tariff plan id
-	Id   string // ActionTimings id
+	Id   string // ActionPlans id
 }
 
 // Queries specific ActionPlan profile on tariff plan
 func (self *ApierV1) GetTPActionPlan(attrs AttrGetTPActionPlan, reply *utils.TPActionPlan) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Id"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ats, err := self.StorDb.GetTPActionTimings(attrs.TPid, attrs.Id); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ats, err := self.StorDb.GetTpActionPlans(attrs.TPid, attrs.Id); err != nil {
+		return utils.NewErrServerError(err)
 	} else if len(ats) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else { // Got the data we need, convert it
-		atRply := &utils.TPActionPlan{attrs.TPid, attrs.Id, ats[attrs.Id]}
+		aps, err := engine.TpActionPlans(ats).GetActionPlans()
+		if err != nil {
+			return err
+		}
+		atRply := &utils.TPActionPlan{
+			TPid:         attrs.TPid,
+			ActionPlanId: attrs.Id,
+			ActionPlan:   aps[attrs.Id],
+		}
 		*reply = *atRply
 	}
 	return nil
@@ -66,17 +75,18 @@ func (self *ApierV1) GetTPActionPlan(attrs AttrGetTPActionPlan, reply *utils.TPA
 
 type AttrGetTPActionPlanIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries ActionPlan identities on specific tariff plan.
 func (self *ApierV1) GetTPActionPlanIds(attrs AttrGetTPActionPlanIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_ACTION_PLANS, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACTION_PLANS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -86,10 +96,10 @@ func (self *ApierV1) GetTPActionPlanIds(attrs AttrGetTPActionPlanIds, reply *[]s
 // Removes specific ActionPlan on Tariff plan
 func (self *ApierV1) RemTPActionPlan(attrs AttrGetTPActionPlan, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Id"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_ACTION_PLANS, attrs.TPid, attrs.Id); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_ACTION_PLANS, attrs.TPid, attrs.Id); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tpactions.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,19 +16,19 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"errors"
 	"fmt"
 
+	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
 // Creates a new Actions profile within a tariff plan
 func (self *ApierV1) SetTPActions(attrs utils.TPActions, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionsId", "Actions"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	for _, action := range attrs.Actions {
 		requiredFields := []string{"Identifier", "Weight"}
@@ -36,11 +36,12 @@ func (self *ApierV1) SetTPActions(attrs utils.TPActions, reply *string) error {
 			requiredFields = append(requiredFields, "Direction", "Units")
 		}
 		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
-			return fmt.Errorf("%s:Action:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
+			return fmt.Errorf("%s:Action:%s:%v", utils.ErrMandatoryIeMissing.Error(), action.Identifier, missing)
 		}
 	}
-	if err := self.StorDb.SetTPActions(attrs.TPid, map[string][]*utils.TPAction{attrs.ActionsId: attrs.Actions}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	as := engine.APItoModelAction(&attrs)
+	if err := self.StorDb.SetTpActions(as); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -54,31 +55,36 @@ type AttrGetTPActions struct {
 // Queries specific Actions profile on tariff plan
 func (self *ApierV1) GetTPActions(attrs AttrGetTPActions, reply *utils.TPActions) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionsId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	if acts, err := self.StorDb.GetTpActions(attrs.TPid, attrs.ActionsId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	} else if len(acts) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
-		*reply = utils.TPActions{TPid: attrs.TPid, ActionsId: attrs.ActionsId, Actions: acts[attrs.ActionsId]}
+		as, err := engine.TpActions(acts).GetActions()
+		if err != nil {
+
+		}
+		*reply = utils.TPActions{TPid: attrs.TPid, ActionsId: attrs.ActionsId, Actions: as[attrs.ActionsId]}
 	}
 	return nil
 }
 
 type AttrGetTPActionIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries Actions identities on specific tariff plan.
 func (self *ApierV1) GetTPActionIds(attrs AttrGetTPActionIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_ACTIONS, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACTIONS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -88,10 +94,10 @@ func (self *ApierV1) GetTPActionIds(attrs AttrGetTPActionIds, reply *[]string) e
 // Removes specific Actions on Tariff plan
 func (self *ApierV1) RemTPActions(attrs AttrGetTPActions, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionsId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_ACTIONS, attrs.TPid, attrs.ActionsId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_ACTIONS, attrs.TPid, attrs.ActionsId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tpactiontriggers.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,12 +16,10 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"errors"
-	"fmt"
-
+	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
@@ -29,13 +27,12 @@ import (
 func (self *ApierV1) SetTPActionTriggers(attrs utils.TPActionTriggers, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs,
 		[]string{"TPid", "ActionTriggersId"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	ats := map[string][]*utils.TPActionTrigger{
-		attrs.ActionTriggersId: attrs.ActionTriggers}
 
-	if err := self.StorDb.SetTPActionTriggers(attrs.TPid, ats); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	ats := engine.APItoModelActionTrigger(&attrs)
+	if err := self.StorDb.SetTpActionTriggers(ats); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -49,14 +46,22 @@ type AttrGetTPActionTriggers struct {
 // Queries specific ActionTriggers profile on tariff plan
 func (self *ApierV1) GetTPActionTriggers(attrs AttrGetTPActionTriggers, reply *utils.TPActionTriggers) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionTriggersId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if atsMap, err := self.StorDb.GetTpActionTriggers(attrs.TPid, attrs.ActionTriggersId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if len(atsMap) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+	if ats, err := self.StorDb.GetTpActionTriggers(attrs.TPid, attrs.ActionTriggersId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(ats) == 0 {
+		return utils.ErrNotFound
 	} else {
-		atRply := &utils.TPActionTriggers{attrs.TPid, attrs.ActionTriggersId, atsMap[attrs.ActionTriggersId]}
+		atsMap, err := engine.TpActionTriggers(ats).GetActionTriggers()
+		if err != nil {
+			return err
+		}
+		atRply := &utils.TPActionTriggers{
+			TPid:             attrs.TPid,
+			ActionTriggersId: attrs.ActionTriggersId,
+			ActionTriggers:   atsMap[attrs.ActionTriggersId],
+		}
 		*reply = *atRply
 	}
 	return nil
@@ -64,17 +69,18 @@ func (self *ApierV1) GetTPActionTriggers(attrs AttrGetTPActionTriggers, reply *u
 
 type AttrGetTPActionTriggerIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries ActionTriggers identities on specific tariff plan.
 func (self *ApierV1) GetTPActionTriggerIds(attrs AttrGetTPActionTriggerIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_ACTION_TRIGGERS, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACTION_TRIGGERS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -84,10 +90,10 @@ func (self *ApierV1) GetTPActionTriggerIds(attrs AttrGetTPActionTriggerIds, repl
 // Removes specific ActionTriggers on Tariff plan
 func (self *ApierV1) RemTPActionTriggers(attrs AttrGetTPActionTriggers, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionTriggersId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_ACTION_TRIGGERS, attrs.TPid, attrs.ActionTriggersId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_ACTION_TRIGGERS, attrs.TPid, attrs.ActionTriggersId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tpcdrstats.go

@@ -0,0 +1,103 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new CdrStats profile within a tariff plan
+func (self *ApierV1) SetTPCdrStats(attrs utils.TPCdrStats, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "CdrStatsId", "CdrStats"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	/*for _, action := range attrs.CdrStats {
+		requiredFields := []string{"Identifier", "Weight"}
+		if action.BalanceType != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions
+			requiredFields = append(requiredFields, "Direction", "Units")
+		}
+		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
+			return fmt.Errorf("%s:CdrStat:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
+		}
+	}*/
+	cs := engine.APItoModelCdrStat(&attrs)
+	if err := self.StorDb.SetTpCdrStats(cs); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPCdrStats struct {
+	TPid       string // Tariff plan id
+	CdrStatsId string // CdrStat id
+}
+
+// Queries specific CdrStat on tariff plan
+func (self *ApierV1) GetTPCdrStats(attrs AttrGetTPCdrStats, reply *utils.TPCdrStats) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "CdrStatsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if sgs, err := self.StorDb.GetTpCdrStats(attrs.TPid, attrs.CdrStatsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(sgs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		csMap, err := engine.TpCdrStats(sgs).GetCdrStats()
+		if err != nil {
+			return err
+		}
+		*reply = utils.TPCdrStats{TPid: attrs.TPid, CdrStatsId: attrs.CdrStatsId, CdrStats: csMap[attrs.CdrStatsId]}
+	}
+	return nil
+}
+
+type AttrGetTPCdrStatIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries CdrStats identities on specific tariff plan.
+func (self *ApierV1) GetTPCdrStatsIds(attrs AttrGetTPCdrStatIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_CDR_STATS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific CdrStats on Tariff plan
+func (self *ApierV1) RemTPCdrStats(attrs AttrGetTPCdrStats, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "CdrStatsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_SHARED_GROUPS, attrs.TPid, attrs.CdrStatsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpderivedcharges.go

@@ -0,0 +1,112 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new DerivedCharges profile within a tariff plan
+func (self *ApierV1) SetTPDerivedChargers(attrs utils.TPDerivedChargers, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Direction", "Tenant", "Category", "Account", "Subject"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	/*for _, action := range attrs.DerivedCharges {
+		requiredFields := []string{"Identifier", "Weight"}
+		if action.BalanceType != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions
+			requiredFields = append(requiredFields, "Direction", "Units")
+		}
+		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
+			return fmt.Errorf("%s:DerivedCharge:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
+		}
+	}*/
+	dc := engine.APItoModelDerivedCharger(&attrs)
+	if err := self.StorDb.SetTpDerivedChargers(dc); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPDerivedChargers struct {
+	TPid              string // Tariff plan id
+	DerivedChargersId string // DerivedCharge id
+}
+
+// Queries specific DerivedCharge on tariff plan
+func (self *ApierV1) GetTPDerivedChargers(attrs AttrGetTPDerivedChargers, reply *utils.TPDerivedChargers) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DerivedChargersId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpDc := &utils.TPDerivedChargers{TPid: attrs.TPid}
+	if err := tmpDc.SetDerivedChargersId(attrs.DerivedChargersId); err != nil {
+		return err
+	}
+	dcs := engine.APItoModelDerivedCharger(tmpDc)
+	if sgs, err := self.StorDb.GetTpDerivedChargers(&dcs[0]); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(sgs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		dcsMap, err := engine.TpDerivedChargers(dcs).GetDerivedChargers()
+		if err != nil {
+			return err
+		}
+		*reply = *dcsMap[attrs.DerivedChargersId]
+	}
+	return nil
+}
+
+type AttrGetTPDerivedChargeIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries DerivedCharges identities on specific tariff plan.
+func (self *ApierV1) GetTPDerivedChargerIds(attrs AttrGetTPDerivedChargeIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_DERIVED_CHARGERS, utils.TPDistinctIds{"loadid", "direction", "tenant", "category", "account", "subject"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific DerivedCharges on Tariff plan
+func (self *ApierV1) RemTPDerivedChargers(attrs AttrGetTPDerivedChargers, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DerivedChargesId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpDc := engine.TpDerivedCharger{}
+	if err := tmpDc.SetDerivedChargersId(attrs.DerivedChargersId); err != nil {
+		return err
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_DERIVED_CHARGERS, attrs.TPid, tmpDc.Loadid, tmpDc.Direction, tmpDc.Tenant, tmpDc.Category, tmpDc.Account, tmpDc.Subject); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpdestinationrates.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,24 +16,23 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 // This file deals with tp_destination_rates management over APIs
 
 import (
-	"errors"
-	"fmt"
-
+	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
 // Creates a new DestinationRate profile within a tariff plan
 func (self *ApierV1) SetTPDestinationRate(attrs utils.TPDestinationRate, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationRateId", "DestinationRates"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.SetTPDestinationRates(attrs.TPid, map[string][]*utils.DestinationRate{attrs.DestinationRateId: attrs.DestinationRates}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	drs := engine.APItoModelDestinationRate(&attrs)
+	if err := self.StorDb.SetTpDestinationRates(drs); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -42,36 +41,42 @@ func (self *ApierV1) SetTPDestinationRate(attrs utils.TPDestinationRate, reply *
 type AttrGetTPDestinationRate struct {
 	TPid              string // Tariff plan id
 	DestinationRateId string // Rate id
+	utils.Paginator
 }
 
 // Queries specific DestinationRate profile on tariff plan
 func (self *ApierV1) GetTPDestinationRate(attrs AttrGetTPDestinationRate, reply *utils.TPDestinationRate) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationRateId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if drs, err := self.StorDb.GetTpDestinationRates(attrs.TPid, attrs.DestinationRateId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if drs, err := self.StorDb.GetTpDestinationRates(attrs.TPid, attrs.DestinationRateId, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if len(drs) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
-		*reply = *drs[attrs.DestinationRateId]
+		drsMap, err := engine.TpDestinationRates(drs).GetDestinationRates()
+		if err != nil {
+			return err
+		}
+		*reply = *drsMap[attrs.DestinationRateId]
 	}
 	return nil
 }
 
 type AttrTPDestinationRateIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries DestinationRate identities on specific tariff plan.
 func (self *ApierV1) GetTPDestinationRateIds(attrs AttrGetTPRateIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_DESTINATION_RATES, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_DESTINATION_RATES, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -81,10 +86,10 @@ func (self *ApierV1) GetTPDestinationRateIds(attrs AttrGetTPRateIds, reply *[]st
 // Removes specific DestinationRate on Tariff plan
 func (self *ApierV1) RemTPDestinationRate(attrs AttrGetTPDestinationRate, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationRateId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_DESTINATION_RATES, attrs.TPid, attrs.DestinationRateId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_DESTINATION_RATES, attrs.TPid, attrs.DestinationRateId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tpdestinations.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,12 +16,9 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"errors"
-	"fmt"
-
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
@@ -29,10 +26,11 @@ import (
 // Creates a new destination within a tariff plan
 func (self *ApierV1) SetTPDestination(attrs utils.TPDestination, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationId", "Prefixes"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.SetTPDestination(attrs.TPid, &engine.Destination{Id: attrs.DestinationId, Prefixes: attrs.Prefixes}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	ds := engine.APItoModelDestination(&attrs)
+	if err := self.StorDb.SetTpDestinations(ds); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -46,31 +44,40 @@ type AttrGetTPDestination struct {
 // Queries a specific destination
 func (self *ApierV1) GetTPDestination(attrs AttrGetTPDestination, reply *utils.TPDestination) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if dsts, err := self.StorDb.GetTpDestinations(attrs.TPid, attrs.DestinationId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
-	} else if len(dsts) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+	if storData, err := self.StorDb.GetTpDestinations(attrs.TPid, attrs.DestinationId); err != nil {
+
+		return utils.NewErrServerError(err)
+	} else if len(storData) == 0 {
+		return utils.ErrNotFound
 	} else {
-		*reply = utils.TPDestination{attrs.TPid, dsts[0].Id, dsts[0].Prefixes}
+		dsts, err := engine.TpDestinations(storData).GetDestinations()
+		if err != nil {
+			return err
+		}
+		*reply = utils.TPDestination{
+			TPid:          attrs.TPid,
+			DestinationId: dsts[attrs.DestinationId].Id,
+			Prefixes:      dsts[attrs.DestinationId].Prefixes}
 	}
 	return nil
 }
 
 type AttrGetTPDestinationIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries destination identities on specific tariff plan.
 func (self *ApierV1) GetTPDestinationIds(attrs AttrGetTPDestinationIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_DESTINATIONS, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_DESTINATIONS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -79,10 +86,10 @@ func (self *ApierV1) GetTPDestinationIds(attrs AttrGetTPDestinationIds, reply *[
 
 func (self *ApierV1) RemTPDestination(attrs AttrGetTPDestination, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_DESTINATIONS, attrs.TPid, attrs.DestinationId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_DESTINATIONS, attrs.TPid, attrs.DestinationId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tplcrrules.go

@@ -0,0 +1,92 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+/*
+// Creates a new LcrRules profile within a tariff plan
+func (self *ApierV1) SetTPLcrRules(attrs utils.TPLcrRules, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LcrRulesId", "LcrRules"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	for _, action := range attrs.LcrRules {
+		requiredFields := []string{"Identifier", "Weight"}
+
+		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
+			return fmt.Errorf("%s:LcrAction:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
+		}
+	}
+	if err := self.StorDb.SetTPLcrRules(attrs.TPid, map[string][]*utils.TPLcrRule{attrs.LcrRulesId: attrs.LcrRules}); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPLcrRules struct {
+	TPid  string // Tariff plan id
+	LcrId string // Lcr id
+}
+
+// Queries specific LcrRules profile on tariff plan
+func (self *ApierV1) GetTPLcrRules(attrs AttrGetTPLcrRules, reply *utils.TPLcrRules) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LcrId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if lcrs, err := self.StorDb.GetTpLCRs(attrs.TPid, attrs.LcrId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(acts) == 0 {
+		return utils.ErrNotFound
+	} else {
+		*reply = utils.TPLcrRules{TPid: attrs.TPid, LcrRulesId: attrs.LcrRulesId, LcrRules: lcrs[attrs.LcrRulesId]}
+	}
+	return nil
+}
+
+type AttrGetTPLcrActionIds struct {
+	TPid string // Tariff plan id
+}
+
+// Queries LcrRules identities on specific tariff plan.
+func (self *ApierV1) GetTPLcrActionIds(attrs AttrGetTPLcrActionIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_LCRS, "id", nil); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific LcrRules on Tariff plan
+func (self *ApierV1) RemTPLcrRules(attrs AttrGetTPLcrRules, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LcrRulesId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTPData(utils.TBL_TP_LCRS, attrs.TPid, attrs.LcrRulesId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}
+*/

apier/v1/tprates.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,24 +16,23 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 // This file deals with tp_rates management over APIs
 
 import (
-	"errors"
-	"fmt"
-
+	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
 // Creates a new rate within a tariff plan
 func (self *ApierV1) SetTPRate(attrs utils.TPRate, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RateId", "RateSlots"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.SetTPRates(attrs.TPid, map[string][]*utils.RateSlot{attrs.RateId: attrs.RateSlots}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	r := engine.APItoModelRate(&attrs)
+	if err := self.StorDb.SetTpRates(r); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -47,31 +46,36 @@ type AttrGetTPRate struct {
 // Queries specific Rate on tariff plan
 func (self *ApierV1) GetTPRate(attrs AttrGetTPRate, reply *utils.TPRate) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RateId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	if rts, err := self.StorDb.GetTpRates(attrs.TPid, attrs.RateId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	} else if len(rts) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
-		*reply = *rts[attrs.RateId]
+		rtsMap, err := engine.TpRates(rts).GetRates()
+		if err != nil {
+			return err
+		}
+		*reply = *rtsMap[attrs.RateId]
 	}
 	return nil
 }
 
 type AttrGetTPRateIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries rate identities on specific tariff plan.
 func (self *ApierV1) GetTPRateIds(attrs AttrGetTPRateIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_RATES, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_RATES, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -81,10 +85,10 @@ func (self *ApierV1) GetTPRateIds(attrs AttrGetTPRateIds, reply *[]string) error
 // Removes specific Rate on Tariff plan
 func (self *ApierV1) RemTPRate(attrs AttrGetTPRate, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RateId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_RATES, attrs.TPid, attrs.RateId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_RATES, attrs.TPid, attrs.RateId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tpratingplans.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,24 +16,23 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 // This file deals with tp_destrates_timing management over APIs
 
 import (
-	"errors"
-	"fmt"
-
+	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
 // Creates a new DestinationRateTiming profile within a tariff plan
 func (self *ApierV1) SetTPRatingPlan(attrs utils.TPRatingPlan, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingPlanId", "RatingPlanBindings"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.SetTPRatingPlans(attrs.TPid, map[string][]*utils.TPRatingPlanBinding{attrs.RatingPlanId: attrs.RatingPlanBindings}); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	rp := engine.APItoModelRatingPlan(&attrs)
+	if err := self.StorDb.SetTpRatingPlans(rp); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -42,36 +41,42 @@ func (self *ApierV1) SetTPRatingPlan(attrs utils.TPRatingPlan, reply *string) er
 type AttrGetTPRatingPlan struct {
 	TPid         string // Tariff plan id
 	RatingPlanId string // Rate id
+	utils.Paginator
 }
 
 // Queries specific RatingPlan profile on tariff plan
 func (self *ApierV1) GetTPRatingPlan(attrs AttrGetTPRatingPlan, reply *utils.TPRatingPlan) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingPlanId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if rps, err := self.StorDb.GetTpRatingPlans(attrs.TPid, attrs.RatingPlanId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if rps, err := self.StorDb.GetTpRatingPlans(attrs.TPid, attrs.RatingPlanId, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if len(rps) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
-		*reply = utils.TPRatingPlan{TPid: attrs.TPid, RatingPlanId: attrs.RatingPlanId, RatingPlanBindings: rps[attrs.RatingPlanId]}
+		rpsMap, err := engine.TpRatingPlans(rps).GetRatingPlans()
+		if err != nil {
+			return err
+		}
+		*reply = utils.TPRatingPlan{TPid: attrs.TPid, RatingPlanId: attrs.RatingPlanId, RatingPlanBindings: rpsMap[attrs.RatingPlanId]}
 	}
 	return nil
 }
 
 type AttrGetTPRatingPlanIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries RatingPlan identities on specific tariff plan.
 func (self *ApierV1) GetTPRatingPlanIds(attrs AttrGetTPRatingPlanIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_RATING_PLANS, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_RATING_PLANS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -81,10 +86,10 @@ func (self *ApierV1) GetTPRatingPlanIds(attrs AttrGetTPRatingPlanIds, reply *[]s
 // Removes specific RatingPlan on Tariff plan
 func (self *ApierV1) RemTPRatingPlan(attrs AttrGetTPRatingPlan, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingPlanId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_RATING_PLANS, attrs.TPid, attrs.RatingPlanId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_RATING_PLANS, attrs.TPid, attrs.RatingPlanId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tpratingprofiles.go

@@ -0,0 +1,170 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+// This file deals with tp_rate_profiles management over APIs
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new RatingProfile within a tariff plan
+func (self *ApierV1) SetTPRatingProfile(attrs utils.TPRatingProfile, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "Category", "Direction", "Subject", "RatingPlanActivations"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	rpf := engine.APItoModelRatingProfile(&attrs)
+	if err := self.StorDb.SetTpRatingProfiles(rpf); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPRatingProfileByLoadId struct {
+	TPid   string // Tariff plan id
+	LoadId string // RatingProfile id
+}
+
+// Queries specific RatingProfile on tariff plan
+func (self *ApierV1) GetTPRatingProfilesByLoadId(attrs utils.TPRatingProfile, reply *[]*utils.TPRatingProfile) error {
+	mndtryFlds := []string{"TPid", "LoadId"}
+	if len(attrs.Subject) != 0 { // If Subject provided as filter, make all related fields mandatory
+		mndtryFlds = append(mndtryFlds, "Tenant", "TOR", "Direction", "Subject")
+	}
+	if missing := utils.MissingStructFields(&attrs, mndtryFlds); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	rpf := engine.APItoModelRatingProfile(&attrs)
+	if dr, err := self.StorDb.GetTpRatingProfiles(&rpf[0]); err != nil {
+		return utils.NewErrServerError(err)
+	} else if dr == nil {
+		return utils.ErrNotFound
+	} else {
+		rpfMap, err := engine.TpRatingProfiles(dr).GetRatingProfiles()
+		if err != nil {
+			return err
+		}
+		var rpfs []*utils.TPRatingProfile
+		if len(attrs.Subject) != 0 {
+			rpfs = []*utils.TPRatingProfile{rpfMap[attrs.KeyId()]}
+		} else {
+			for _, rpfLst := range rpfMap {
+				rpfs = append(rpfs, rpfLst)
+			}
+		}
+		*reply = rpfs
+	}
+	return nil
+}
+
+// Queries RatingProfile identities on specific tariff plan.
+func (self *ApierV1) GetTPRatingProfileLoadIds(attrs utils.AttrTPRatingProfileIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_RATE_PROFILES, utils.TPDistinctIds{"loadid"}, map[string]string{
+		"tenant":    attrs.Tenant,
+		"tor":       attrs.Category,
+		"direction": attrs.Direction,
+		"subject":   attrs.Subject,
+	}, new(utils.Paginator)); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+type AttrGetTPRatingProfile struct {
+	TPid            string // Tariff plan id
+	RatingProfileId string // RatingProfile id
+}
+
+// Queries specific RatingProfile on tariff plan
+func (self *ApierV1) GetTPRatingProfile(attrs AttrGetTPRatingProfile, reply *utils.TPRatingProfile) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingProfileId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpRpf := &utils.TPRatingProfile{TPid: attrs.TPid}
+	if err := tmpRpf.SetRatingProfilesId(attrs.RatingProfileId); err != nil {
+		return err
+	}
+	rpf := engine.APItoModelRatingProfile(tmpRpf)
+	if rpfs, err := self.StorDb.GetTpRatingProfiles(&rpf[0]); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(rpfs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		rpfMap, err := engine.TpRatingProfiles(rpfs).GetRatingProfiles()
+		if err != nil {
+			return err
+		}
+		rpf := rpfMap[tmpRpf.KeyId()]
+		tpdc := utils.TPRatingProfile{
+			TPid: attrs.TPid,
+			RatingPlanActivations: rpf.RatingPlanActivations,
+		}
+		if err := tpdc.SetRatingProfilesId(attrs.RatingProfileId); err != nil {
+			return err
+		}
+		*reply = tpdc
+	}
+	return nil
+}
+
+type AttrGetTPRatingProfileIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries RatingProfiles identities on specific tariff plan.
+func (self *ApierV1) GetTPRatingProfileIds(attrs AttrGetTPRatingProfileIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_RATE_PROFILES, utils.TPDistinctIds{"loadid", "direction", "tenant", "category", "subject"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific RatingProfiles on Tariff plan
+func (self *ApierV1) RemTPRatingProfile(attrs AttrGetTPRatingProfile, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingProfileId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpRpf := engine.TpRatingProfile{}
+	if err := tmpRpf.SetRatingProfileId(attrs.RatingProfileId); err != nil {
+		return err
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_RATE_PROFILES, attrs.TPid, tmpRpf.Loadid, tmpRpf.Direction, tmpRpf.Tenant, tmpRpf.Category, tmpRpf.Subject); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpsharedgroups.go

@@ -0,0 +1,103 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new SharedGroups profile within a tariff plan
+func (self *ApierV1) SetTPSharedGroups(attrs utils.TPSharedGroups, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "SharedGroupsId", "SharedGroups"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	/*for _, action := range attrs.SharedGroups {
+		requiredFields := []string{"Identifier", "Weight"}
+		if action.BalanceType != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions
+			requiredFields = append(requiredFields, "Direction", "Units")
+		}
+		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
+			return fmt.Errorf("%s:SharedGroup:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
+		}
+	}*/
+	sg := engine.APItoModelSharedGroup(&attrs)
+	if err := self.StorDb.SetTpSharedGroups(sg); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPSharedGroups struct {
+	TPid           string // Tariff plan id
+	SharedGroupsId string // SharedGroup id
+}
+
+// Queries specific SharedGroup on tariff plan
+func (self *ApierV1) GetTPSharedGroups(attrs AttrGetTPSharedGroups, reply *utils.TPSharedGroups) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "SharedGroupsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if sgs, err := self.StorDb.GetTpSharedGroups(attrs.TPid, attrs.SharedGroupsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(sgs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		sgMap, err := engine.TpSharedGroups(sgs).GetSharedGroups()
+		if err != nil {
+			return err
+		}
+		*reply = utils.TPSharedGroups{TPid: attrs.TPid, SharedGroupsId: attrs.SharedGroupsId, SharedGroups: sgMap[attrs.SharedGroupsId]}
+	}
+	return nil
+}
+
+type AttrGetTPSharedGroupIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries SharedGroups identities on specific tariff plan.
+func (self *ApierV1) GetTPSharedGroupIds(attrs AttrGetTPSharedGroupIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_SHARED_GROUPS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific SharedGroups on Tariff plan
+func (self *ApierV1) RemTPSharedGroups(attrs AttrGetTPSharedGroups, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "SharedGroupsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_SHARED_GROUPS, attrs.TPid, attrs.SharedGroupsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tptimings.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,12 +16,9 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package apier
+package v1
 
 import (
-	"errors"
-	"fmt"
-
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
@@ -29,11 +26,11 @@ import (
 // Creates a new timing within a tariff plan
 func (self *ApierV1) SetTPTiming(attrs utils.ApierTPTiming, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "TimingId", "Years", "Months", "MonthDays", "WeekDays", "Time"}); len(missing) != 0 {
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	tm := engine.NewTiming(attrs.TimingId, attrs.Years, attrs.Months, attrs.MonthDays, attrs.WeekDays, attrs.Time)
-	if err := self.StorDb.SetTPTiming(attrs.TPid, tm); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	tm := engine.APItoModelTiming(&attrs)
+	if err := self.StorDb.SetTpTimings([]engine.TpTiming{*tm}); err != nil {
+		return utils.NewErrServerError(err)
 	}
 	*reply = "OK"
 	return nil
@@ -47,33 +44,36 @@ type AttrGetTPTiming struct {
 // Queries specific Timing on Tariff plan
 func (self *ApierV1) GetTPTiming(attrs AttrGetTPTiming, reply *utils.ApierTPTiming) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "TimingId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
 	if tms, err := self.StorDb.GetTpTimings(attrs.TPid, attrs.TimingId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+		return utils.NewErrServerError(err)
 	} else if len(tms) == 0 {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
-		tm := tms[attrs.TimingId]
-		*reply = utils.ApierTPTiming{attrs.TPid, tm.Id, tm.Years.Serialize(";"),
-			tm.Months.Serialize(";"), tm.MonthDays.Serialize(";"), tm.WeekDays.Serialize(";"), tm.StartTime}
+		tmMap, err := engine.TpTimings(tms).GetApierTimings()
+		if err != nil {
+			return err
+		}
+		*reply = *tmMap[attrs.TimingId]
 	}
 	return nil
 }
 
 type AttrGetTPTimingIds struct {
 	TPid string // Tariff plan id
+	utils.Paginator
 }
 
 // Queries timing identities on specific tariff plan.
 func (self *ApierV1) GetTPTimingIds(attrs AttrGetTPTimingIds, reply *[]string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if ids, err := self.StorDb.GetTPTableIds(attrs.TPid, utils.TBL_TP_TIMINGS, "id", nil); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_TIMINGS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
 	} else if ids == nil {
-		return errors.New(utils.ERR_NOT_FOUND)
+		return utils.ErrNotFound
 	} else {
 		*reply = ids
 	}
@@ -83,10 +83,10 @@ func (self *ApierV1) GetTPTimingIds(attrs AttrGetTPTimingIds, reply *[]string) e
 // Removes specific Timing on Tariff plan
 func (self *ApierV1) RemTPTiming(attrs AttrGetTPTiming, reply *string) error {
 	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "TimingId"}); len(missing) != 0 { //Params missing
-		return fmt.Errorf("%s:%v", utils.ERR_MANDATORY_IE_MISSING, missing)
+		return utils.NewErrMandatoryIeMissing(missing...)
 	}
-	if err := self.StorDb.RemTPData(utils.TBL_TP_TIMINGS, attrs.TPid, attrs.TimingId); err != nil {
-		return fmt.Errorf("%s:%s", utils.ERR_SERVER_ERROR, err.Error())
+	if err := self.StorDb.RemTpData(utils.TBL_TP_TIMINGS, attrs.TPid, attrs.TimingId); err != nil {
+		return utils.NewErrServerError(err)
 	} else {
 		*reply = "OK"
 	}

apier/v1/tutfsjson_local_test

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -41,7 +41,7 @@ var waitFs = flag.Int("wait_fs", 500, "Number of miliseconds to wait for FreeSWI
 
 func init() {
 	fsjsonCfgPath = path.Join(*dataDir, "tutorials", "fs_json", "cgrates", "etc", "cgrates", "cgrates.cfg")
-	fsjsonCfg, _ = config.NewCGRConfig(&fsjsonCfgPath)
+	fsjsonCfg, _ = config.NewCGRConfigFromFile(&fsjsonCfgPath)
 }
 
 // Remove here so they can be properly created by init script
@@ -49,7 +49,7 @@ func TestFsJsonRemoveDirs(t *testing.T) {
 	if !*testLocal {
 		return
 	}
-	for _, pathDir := range []string{fsjsonCfg.CdreDir, fsjsonCfg.HistoryDir} {
+	for _, pathDir := range []string{cfg.CdreDefaultInstance.ExportDir, fsjsonCfg.HistoryDir} {
 		if err := os.RemoveAll(pathDir); err != nil {
 			t.Fatal("Error removing folder: ", pathDir, err)
 		}
@@ -70,11 +70,9 @@ func TestFsJsonCreateTables(t *testing.T) {
 	} else {
 		mysql = d.(*engine.MySQLStorage)
 	}
-	for _, scriptName := range []string{engine.CREATE_CDRS_TABLES_SQL, engine.CREATE_COSTDETAILS_TABLES_SQL, engine.CREATE_MEDIATOR_TABLES_SQL} {
-		if err := mysql.CreateTablesFromScript(path.Join(*dataDir, "storage", *storDbType, scriptName)); err != nil {
-			t.Fatal("Error on mysql creation: ", err.Error())
-			return // No point in going further
-		}
+	if err := mysql.CreateTablesFromScript(path.Join(*dataDir, "storage", *storDbType, engine.CREATE_CDRS_TABLES_SQL)); err != nil {
+		t.Fatal("Error on mysql creation: ", err.Error())
+		return // No point in going further
 	}
 	for _, tbl := range []string{utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_EXTRA} {
 		if _, err := mysql.Db.Query(fmt.Sprintf("SELECT 1 from %s", tbl)); err != nil {
@@ -171,7 +169,7 @@ func TestFsJsonLoadTariffPlans(t *testing.T) {
 	}
 	time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups
 	var rcvStats *utils.CacheStats
-	expectedStats := &utils.CacheStats{Destinations: 3, RatingPlans: 2, RatingProfiles: 2, Actions: 5, SharedGroups: 1, RatingAliases: 1, AccountAliases: 1}
+	expectedStats := &utils.CacheStats{Destinations: 3, RatingPlans: 3, RatingProfiles: 3, Actions: 5, SharedGroups: 1, RatingAliases: 1, AccountAliases: 1, DerivedChargers: 1}
 	var args utils.AttrCacheStats
 	if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil {
 		t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())
@@ -185,17 +183,17 @@ func TestFsJsonGetAccount1001(t *testing.T) {
 		return
 	}
 	var acnt *engine.Account
-	attrs := &AttrGetAccount{Tenant: "cgrates.org", Account: "1001", BalanceType: "*monetary", Direction: "*out"}
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001", Direction: "*out"}
 	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
 		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
 	}
-	if acnt.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue() != 10.0 {
-		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", acnt.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue())
+	if acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 10.0 {
+		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
 	}
-	if len(acnt.BalanceMap[attrs.BalanceType+attrs.Direction]) != 2 {
-		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[attrs.BalanceType+attrs.Direction]))
+	if len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]) != 2 {
+		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]))
 	}
-	blncLst := acnt.BalanceMap[attrs.BalanceType+attrs.Direction]
+	blncLst := acnt.BalanceMap[engine.CREDIT+attrs.Direction]
 	for _, blnc := range blncLst {
 		if len(blnc.SharedGroup) == 0 && blnc.Value != 5 {
 			t.Errorf("Unexpected value for general balance: %f", blnc.Value)
@@ -210,17 +208,17 @@ func TestFsJsonGetAccount1002(t *testing.T) {
 		return
 	}
 	var acnt *engine.Account
-	attrs := &AttrGetAccount{Tenant: "cgrates.org", Account: "1002", BalanceType: "*monetary", Direction: "*out"}
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1002", Direction: "*out"}
 	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
 		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
 	}
-	if acnt.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue() != 10.0 {
-		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", acnt.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue())
+	if acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 10.0 {
+		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
 	}
-	if len(acnt.BalanceMap[attrs.BalanceType+attrs.Direction]) != 1 {
-		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[attrs.BalanceType+attrs.Direction]))
+	if len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]) != 1 {
+		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]))
 	}
-	blnc := acnt.BalanceMap[attrs.BalanceType+attrs.Direction][0]
+	blnc := acnt.BalanceMap[engine.CREDIT+attrs.Direction][0]
 	if blnc.Value != 10 {
 		t.Errorf("Unexpected value for general balance: %f", blnc.Value)
 	}
@@ -231,17 +229,17 @@ func TestFsJsonGetAccount1003(t *testing.T) {
 		return
 	}
 	var acnt *engine.Account
-	attrs := &AttrGetAccount{Tenant: "cgrates.org", Account: "1003", BalanceType: "*monetary", Direction: "*out"}
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1003", Direction: "*out"}
 	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
 		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
 	}
-	if acnt.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue() != 10.0 {
-		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", acnt.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue())
+	if acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 10.0 {
+		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
 	}
-	if len(acnt.BalanceMap[attrs.BalanceType+attrs.Direction]) != 1 {
-		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[attrs.BalanceType+attrs.Direction]))
+	if len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]) != 1 {
+		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]))
 	}
-	blnc := acnt.BalanceMap[attrs.BalanceType+attrs.Direction][0]
+	blnc := acnt.BalanceMap[engine.CREDIT+attrs.Direction][0]
 	if blnc.Value != 10 {
 		t.Errorf("Unexpected value for general balance: %f", blnc.Value)
 	}
@@ -252,17 +250,17 @@ func TestFsJsonGetAccount1004(t *testing.T) {
 		return
 	}
 	var acnt *engine.Account
-	attrs := &AttrGetAccount{Tenant: "cgrates.org", Account: "1004", BalanceType: "*monetary", Direction: "*out"}
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1004", Direction: "*out"}
 	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
 		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
 	}
-	if acnt.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue() != 10.0 {
-		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", acnt.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue())
+	if acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 10.0 {
+		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
 	}
-	if len(acnt.BalanceMap[attrs.BalanceType+attrs.Direction]) != 1 {
-		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[attrs.BalanceType+attrs.Direction]))
+	if len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]) != 1 {
+		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]))
 	}
-	blnc := acnt.BalanceMap[attrs.BalanceType+attrs.Direction][0]
+	blnc := acnt.BalanceMap[engine.CREDIT+attrs.Direction][0]
 	if blnc.Value != 10 {
 		t.Errorf("Unexpected value for general balance: %f", blnc.Value)
 	}
@@ -273,7 +271,7 @@ func TestFsJsonGetAccount1006(t *testing.T) {
 		return
 	}
 	var acnt *engine.Account
-	attrs := &AttrGetAccount{Tenant: "cgrates.org", Account: "1006", BalanceType: "*monetary", Direction: "*out"}
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1006", Direction: "*out"}
 	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err == nil {
 		t.Error("Got no error when querying unexisting balance")
 	}
@@ -284,17 +282,17 @@ func TestFsJsonGetAccount1007(t *testing.T) {
 		return
 	}
 	var acnt *engine.Account
-	attrs := &AttrGetAccount{Tenant: "cgrates.org", Account: "1007", BalanceType: "*monetary", Direction: "*out"}
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1007", Direction: "*out"}
 	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
 		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
 	}
-	if acnt.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue() != 0 {
-		t.Errorf("Calling ApierV1.GetBalance expected: 0, received: %f", acnt.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue())
+	if acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 0 {
+		t.Errorf("Calling ApierV1.GetBalance expected: 0, received: %f", acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
 	}
-	if len(acnt.BalanceMap[attrs.BalanceType+attrs.Direction]) != 1 {
-		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[attrs.BalanceType+attrs.Direction]))
+	if len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]) != 1 {
+		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]))
 	}
-	blncLst := acnt.BalanceMap[attrs.BalanceType+attrs.Direction]
+	blncLst := acnt.BalanceMap[engine.CREDIT+attrs.Direction]
 	for _, blnc := range blncLst {
 		if len(blnc.SharedGroup) == 0 && blnc.Value != 0 { // General balance
 			t.Errorf("Unexpected value for general balance: %f", blnc.Value)
@@ -311,12 +309,12 @@ func TestMaxCallDuration(t *testing.T) {
 	cd := engine.CallDescriptor{
 		Direction:   "*out",
 		Tenant:      "cgrates.org",
-		TOR:         "call",
+		Category:    "call",
 		Subject:     "1001",
 		Account:     "1001",
 		Destination: "1002",
-		TimeStart:   time.Now(),
-		TimeEnd:     time.Now().Add(fsjsonCfg.SMMaxCallDuration),
+		TimeStart:   time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC),
+		TimeEnd:     time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC).Add(fsjsonCfg.SMMaxCallDuration),
 	}
 	var remainingDurationFloat float64
 	if err := rater.Call("Responder.GetMaxSessionTime", cd, &remainingDurationFloat); err != nil {
@@ -330,12 +328,12 @@ func TestMaxCallDuration(t *testing.T) {
 	cd = engine.CallDescriptor{
 		Direction:   "*out",
 		Tenant:      "cgrates.org",
-		TOR:         "call",
+		Category:    "call",
 		Subject:     "1002",
 		Account:     "1002",
 		Destination: "1001",
-		TimeStart:   time.Now(),
-		TimeEnd:     time.Now().Add(fsjsonCfg.SMMaxCallDuration),
+		TimeStart:   time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC),
+		TimeEnd:     time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC).Add(fsjsonCfg.SMMaxCallDuration),
 	}
 	if err := rater.Call("Responder.GetMaxSessionTime", cd, &remainingDurationFloat); err != nil {
 		t.Error(err)
@@ -348,12 +346,12 @@ func TestMaxCallDuration(t *testing.T) {
 	cd = engine.CallDescriptor{
 		Direction:   "*out",
 		Tenant:      "cgrates.org",
-		TOR:         "call",
+		Category:    "call",
 		Subject:     "1006",
 		Account:     "1006",
 		Destination: "1001",
-		TimeStart:   time.Now(),
-		TimeEnd:     time.Now().Add(fsjsonCfg.SMMaxCallDuration),
+		TimeStart:   time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC),
+		TimeEnd:     time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC).Add(fsjsonCfg.SMMaxCallDuration),
 	}
 	if err := rater.Call("Responder.GetMaxSessionTime", cd, &remainingDurationFloat); err != nil {
 		t.Error(err)
@@ -367,12 +365,12 @@ func TestMaxCallDuration(t *testing.T) {
 	cd = engine.CallDescriptor{
 		Direction:   "*out",
 		Tenant:      "cgrates.org",
-		TOR:         "call",
+		Category:    "call",
 		Subject:     "1007",
 		Account:     "1007",
 		Destination: "1001",
-		TimeStart:   time.Now(),
-		TimeEnd:     time.Now().Add(fsjsonCfg.SMMaxCallDuration),
+		TimeStart:   time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC),
+		TimeEnd:     time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC).Add(fsjsonCfg.SMMaxCallDuration),
 	}
 	if err := rater.Call("Responder.GetMaxSessionTime", cd, &remainingDurationFloat); err != nil {
 		t.Error(err)
@@ -393,19 +391,19 @@ func TestMaxDebit1001(t *testing.T) {
 	cd := engine.CallDescriptor{
 		Direction:   "*out",
 		Tenant:      "cgrates.org",
-		TOR:         "call",
+		Category:    "call",
 		Subject:     "1001",
 		Account:     "1001",
 		Destination: "1002",
-		TimeStart:   time.Now(),
-		TimeEnd:     time.Now().Add(time.Duration(10) * time.Second),
+		TimeStart:   time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC),
+		TimeEnd:     time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC).Add(time.Duration(10) * time.Second),
 	}
 	if err := rater.Call("Responder.MaxDebit", cd, cc); err != nil {
 		t.Error(err.Error())
 	} else if cc.GetDuration() > time.Duration(1)*time.Minute {
 		t.Errorf("Unexpected call duration received: %v", cc.GetDuration())
 	}
-	attrs := &AttrGetAccount{Tenant: "cgrates.org", Account: "1001", BalanceType: "*monetary", Direction: "*out"}
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001", Direction: "*out"}
 	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
 		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
 	} else {
@@ -432,12 +430,12 @@ func TestMaxDebit1007(t *testing.T) {
 	cd := engine.CallDescriptor{
 		Direction:   "*out",
 		Tenant:      "cgrates.org",
-		TOR:         "call",
+		Category:    "call",
 		Subject:     "1007",
 		Account:     "1007",
 		Destination: "1002",
-		TimeStart:   time.Now(),
-		TimeEnd:     time.Now().Add(time.Duration(10) * time.Second),
+		TimeStart:   time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC),
+		TimeEnd:     time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC).Add(time.Duration(10) * time.Second),
 	}
 	if err := rater.Call("Responder.MaxDebit", cd, cc); err != nil {
 		t.Error(err.Error())
@@ -445,7 +443,7 @@ func TestMaxDebit1007(t *testing.T) {
 		t.Errorf("Unexpected call duration received: %v", cc.GetDuration())
 	}
 	// Debit out of shared balance should reflect in the 1001 instead of 1007
-	attrs := &AttrGetAccount{Tenant: "cgrates.org", Account: "1001", BalanceType: "*monetary", Direction: "*out"}
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001", Direction: "*out"}
 	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
 		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
 	} else {
@@ -462,17 +460,17 @@ func TestMaxDebit1007(t *testing.T) {
 		}
 	}
 	// Make sure 1007 remains the same
-	attrs = &AttrGetAccount{Tenant: "cgrates.org", Account: "1007", BalanceType: "*monetary", Direction: "*out"}
+	attrs = &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1007", Direction: "*out"}
 	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
 		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
 	}
-	if acnt.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue() != 0 {
-		t.Errorf("Calling ApierV1.GetBalance expected: 0, received: %f", acnt.BalanceMap[attrs.BalanceType+attrs.Direction].GetTotalValue())
+	if acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 0 {
+		t.Errorf("Calling ApierV1.GetBalance expected: 0, received: %f", acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
 	}
-	if len(acnt.BalanceMap[attrs.BalanceType+attrs.Direction]) != 1 {
-		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[attrs.BalanceType+attrs.Direction]))
+	if len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]) != 1 {
+		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]))
 	}
-	blnc := acnt.BalanceMap[attrs.BalanceType+attrs.Direction][0]
+	blnc := acnt.BalanceMap[engine.CREDIT+attrs.Direction][0]
 	if len(blnc.SharedGroup) == 0 { // General balance
 		t.Errorf("Unexpected general balance: %f", blnc.Value)
 	} else if blnc.SharedGroup == "SHARED_A" && blnc.Value != 0 {
@@ -480,6 +478,30 @@ func TestMaxDebit1007(t *testing.T) {
 	}
 }
 
+func TestDerivedChargers1001(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	attrs := &utils.AttrDerivedChargers{Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001"}
+	expectedDCs := utils.DerivedChargers([]*utils.DerivedCharger{
+		&utils.DerivedCharger{RunId: "fs_json_run", ReqTypeField: "^rated", DirectionField: "*default", TenantField: "*default", CategoryField: "*default",
+			AccountField: "*default", SubjectField: "^1002", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"},
+	})
+	var rcvRspDCs utils.DerivedChargers
+	if err := rater.Call("Responder.GetDerivedChargers", attrs, &rcvRspDCs); err != nil {
+		t.Error(err.Error())
+	} else if !reflect.DeepEqual(expectedDCs, rcvRspDCs) {
+		t.Errorf("Expected: %v: received: %v", expectedDCs, rcvRspDCs)
+	}
+	// Make sure that over Apier we get the same result as over Responder
+	var rcvApierDCs utils.DerivedChargers
+	if err := rater.Call("ApierV1.GetDerivedChargers", attrs, &rcvApierDCs); err != nil {
+		t.Error(err.Error())
+	} else if !reflect.DeepEqual(rcvRspDCs, rcvApierDCs) {
+		t.Errorf("Expected: %v: received: %v", rcvRspDCs, rcvApierDCs)
+	}
+}
+
 // Simply kill the engine after we are done with tests within this file
 func TestFsJsonStopEngine(t *testing.T) {
 	if !*testLocal {

apier/v2/apier.go

@@ -0,0 +1,122 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"github.com/cgrates/cgrates/apier/v1"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type ApierV2 struct {
+	v1.ApierV1
+}
+
+type AttrLoadRatingProfile struct {
+	TPid            string
+	RatingProfileId string
+}
+
+// Process dependencies and load a specific rating profile from storDb into dataDb.
+func (self *ApierV2) LoadRatingProfile(attrs AttrLoadRatingProfile, reply *string) error {
+	if len(attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	tpRpf := &utils.TPRatingProfile{TPid: attrs.TPid}
+	tpRpf.SetRatingProfilesId(attrs.RatingProfileId)
+	rpf := engine.APItoModelRatingProfile(tpRpf)
+	dbReader := engine.NewTpReader(self.RatingDb, self.AccountDb, self.StorDb, attrs.TPid)
+	if err := dbReader.LoadRatingProfilesFiltered(&rpf[0]); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	//Automatic cache of the newly inserted rating profile
+	var ratingProfile []string
+	if tpRpf.KeyId() != ":::" { // if has some filters
+		ratingProfile = []string{utils.RATING_PROFILE_PREFIX + tpRpf.KeyId()}
+	}
+	if err := self.RatingDb.CachePrefixValues(map[string][]string{utils.RATING_PROFILE_PREFIX: ratingProfile}); err != nil {
+		return err
+	}
+	*reply = v1.OK
+	return nil
+}
+
+type AttrLoadAccountActions struct {
+	TPid             string
+	AccountActionsId string
+}
+
+// Process dependencies and load a specific AccountActions profile from storDb into dataDb.
+func (self *ApierV2) LoadAccountActions(attrs AttrLoadAccountActions, reply *string) error {
+	if len(attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	dbReader := engine.NewTpReader(self.RatingDb, self.AccountDb, self.StorDb, attrs.TPid)
+	tpAa := &utils.TPAccountActions{TPid: attrs.TPid}
+	tpAa.SetAccountActionsId(attrs.AccountActionsId)
+	aa := engine.APItoModelAccountAction(tpAa)
+	if _, err := engine.Guardian.Guard(func() (interface{}, error) {
+		if err := dbReader.LoadAccountActionsFiltered(aa); err != nil {
+			return 0, err
+		}
+		return 0, nil
+	}, attrs.AccountActionsId); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	// ToDo: Get the action keys loaded by dbReader so we reload only these in cache
+	// Need to do it before scheduler otherwise actions to run will be unknown
+	if err := self.RatingDb.CachePrefixes(utils.DERIVED_CHARGERS_CSV, utils.ACTION_PREFIX, utils.SHARED_GROUP_PREFIX, utils.ACC_ALIAS_PREFIX); err != nil {
+		return err
+	}
+	if self.Sched != nil {
+		self.Sched.LoadActionPlans(self.RatingDb)
+		self.Sched.Restart()
+	}
+	*reply = v1.OK
+	return nil
+}
+
+type AttrLoadDerivedChargers struct {
+	TPid              string
+	DerivedChargersId string
+}
+
+// Load derived chargers from storDb into dataDb.
+func (self *ApierV2) LoadDerivedChargers(attrs AttrLoadDerivedChargers, reply *string) error {
+	if len(attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	tpDc := &utils.TPDerivedChargers{TPid: attrs.TPid}
+	tpDc.SetDerivedChargersId(attrs.DerivedChargersId)
+	dc := engine.APItoModelDerivedCharger(tpDc)
+	dbReader := engine.NewTpReader(self.RatingDb, self.AccountDb, self.StorDb, attrs.TPid)
+	if err := dbReader.LoadDerivedChargersFiltered(&dc[0], true); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	//Automatic cache of the newly inserted rating plan
+	var dcsChanged []string
+	if len(attrs.DerivedChargersId) != 0 {
+		dcsChanged = []string{utils.DERIVEDCHARGERS_PREFIX + attrs.DerivedChargersId}
+	}
+	if err := self.RatingDb.CachePrefixValues(map[string][]string{utils.DERIVEDCHARGERS_PREFIX: dcsChanged}); err != nil {
+		return err
+	}
+	*reply = v1.OK
+	return nil
+}

apier/v2/cdre.go

@@ -0,0 +1,134 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/cgrates/cgrates/cdre"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Export Cdrs to file
+func (self *ApierV2) ExportCdrsToFile(attr utils.AttrExportCdrsToFile, reply *utils.ExportedFileCdrs) error {
+	var err error
+	exportTemplate := self.Config.CdreProfiles[utils.META_DEFAULT]
+	if attr.ExportTemplate != nil && len(*attr.ExportTemplate) != 0 { // Export template prefered, use it
+		var hasIt bool
+		if exportTemplate, hasIt = self.Config.CdreProfiles[*attr.ExportTemplate]; !hasIt {
+			return fmt.Errorf("%s:ExportTemplate", utils.ErrNotFound)
+		}
+	}
+	cdrFormat := exportTemplate.CdrFormat
+	if attr.CdrFormat != nil && len(*attr.CdrFormat) != 0 {
+		cdrFormat = strings.ToLower(*attr.CdrFormat)
+	}
+	if !utils.IsSliceMember(utils.CdreCdrFormats, cdrFormat) {
+		return utils.NewErrMandatoryIeMissing("CdrFormat")
+	}
+	fieldSep := exportTemplate.FieldSeparator
+	if attr.FieldSeparator != nil && len(*attr.FieldSeparator) != 0 {
+		fieldSep, _ = utf8.DecodeRuneInString(*attr.FieldSeparator)
+		if fieldSep == utf8.RuneError {
+			return fmt.Errorf("%s:FieldSeparator:%s", utils.ErrServerError, "Invalid")
+		}
+	}
+	exportDir := exportTemplate.ExportDir
+	if attr.ExportDir != nil && len(*attr.ExportDir) != 0 {
+		exportDir = *attr.ExportDir
+	}
+	exportId := strconv.FormatInt(time.Now().Unix(), 10)
+	if attr.ExportId != nil && len(*attr.ExportId) != 0 {
+		exportId = *attr.ExportId
+	}
+	fileName := fmt.Sprintf("cdre_%s.%s", exportId, cdrFormat)
+	if attr.ExportFileName != nil && len(*attr.ExportFileName) != 0 {
+		fileName = *attr.ExportFileName
+	}
+	filePath := path.Join(exportDir, fileName)
+	if cdrFormat == utils.DRYRUN {
+		filePath = utils.DRYRUN
+	}
+	dataUsageMultiplyFactor := exportTemplate.DataUsageMultiplyFactor
+	if attr.DataUsageMultiplyFactor != nil && *attr.DataUsageMultiplyFactor != 0.0 {
+		dataUsageMultiplyFactor = *attr.DataUsageMultiplyFactor
+	}
+	smsUsageMultiplyFactor := exportTemplate.SmsUsageMultiplyFactor
+	if attr.SmsUsageMultiplyFactor != nil && *attr.SmsUsageMultiplyFactor != 0.0 {
+		smsUsageMultiplyFactor = *attr.SmsUsageMultiplyFactor
+	}
+	genericUsageMultiplyFactor := exportTemplate.GenericUsageMultiplyFactor
+	if attr.GenericUsageMultiplyFactor != nil && *attr.GenericUsageMultiplyFactor != 0.0 {
+		genericUsageMultiplyFactor = *attr.GenericUsageMultiplyFactor
+	}
+	costMultiplyFactor := exportTemplate.CostMultiplyFactor
+	if attr.CostMultiplyFactor != nil && *attr.CostMultiplyFactor != 0.0 {
+		costMultiplyFactor = *attr.CostMultiplyFactor
+	}
+	costShiftDigits := exportTemplate.CostShiftDigits
+	if attr.CostShiftDigits != nil {
+		costShiftDigits = *attr.CostShiftDigits
+	}
+	roundingDecimals := exportTemplate.CostRoundingDecimals
+	if attr.RoundDecimals != nil {
+		roundingDecimals = *attr.RoundDecimals
+	}
+	maskDestId := exportTemplate.MaskDestId
+	if attr.MaskDestinationId != nil && len(*attr.MaskDestinationId) != 0 {
+		maskDestId = *attr.MaskDestinationId
+	}
+	maskLen := exportTemplate.MaskLength
+	if attr.MaskLength != nil {
+		maskLen = *attr.MaskLength
+	}
+	cdrsFltr, err := attr.RpcCdrsFilter.AsCdrsFilter()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	cdrs, _, err := self.CdrDb.GetStoredCdrs(cdrsFltr)
+	if err != nil {
+		return err
+	} else if len(cdrs) == 0 {
+		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
+		return nil
+	}
+	cdrexp, err := cdre.NewCdrExporter(cdrs, self.CdrDb, exportTemplate, cdrFormat, fieldSep, exportId, dataUsageMultiplyFactor, smsUsageMultiplyFactor, genericUsageMultiplyFactor,
+		costMultiplyFactor, costShiftDigits, roundingDecimals, self.Config.RoundingDecimals, maskDestId, maskLen, self.Config.HttpSkipTlsVerify)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if cdrexp.TotalExportedCdrs() == 0 {
+		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
+		return nil
+	}
+	if err := cdrexp.WriteToFile(filePath); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), TotalCost: cdrexp.TotalCost(), FirstOrderId: cdrexp.FirstOrderId(), LastOrderId: cdrexp.LastOrderId()}
+	if !attr.SuppressCgrIds {
+		reply.ExportedCgrIds = cdrexp.PositiveExports()
+		reply.UnexportedCgrIds = cdrexp.NegativeExports()
+	}
+	return nil
+}

apier/v2/cdrs.go

@@ -0,0 +1,63 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"github.com/cgrates/cgrates/apier/v1"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Retrieves CDRs based on the filters
+func (apier *ApierV2) GetCdrs(attrs utils.RpcCdrsFilter, reply *[]*engine.ExternalCdr) error {
+	cdrsFltr, err := attrs.AsCdrsFilter()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if cdrs, _, err := apier.CdrDb.GetStoredCdrs(cdrsFltr); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(cdrs) == 0 {
+		*reply = make([]*engine.ExternalCdr, 0)
+	} else {
+		for _, cdr := range cdrs {
+			*reply = append(*reply, cdr.AsExternalCdr())
+		}
+	}
+	return nil
+}
+
+func (apier *ApierV2) CountCdrs(attrs utils.RpcCdrsFilter, reply *int64) error {
+	cdrsFltr, err := attrs.AsCdrsFilter()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	cdrsFltr.Count = true
+	if _, count, err := apier.CdrDb.GetStoredCdrs(cdrsFltr); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = count
+	}
+	return nil
+}
+
+// Receive CDRs via RPC methods, not included with APIer because it has way less dependencies and can be standalone
+type CdrsV2 struct {
+	v1.CdrsV1
+}

apier/v2/cdrs_mysql_local_test.go

@@ -0,0 +1,250 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"flag"
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"path"
+	"testing"
+	"time"
+)
+
+var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args
+var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here")
+var waitRater = flag.Int("wait_rater", 500, "Number of miliseconds to wait for rater to start and cache")
+
+var cdrsCfgPath string
+var cdrsCfg *config.CGRConfig
+var cdrsRpc *rpc.Client
+
+func TestV2CdrsMysqlInitConfig(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrsCfgPath = path.Join(*dataDir, "conf", "samples", "cdrsv2mysql")
+	if cdrsCfg, err = config.NewCGRConfigFromFolder(cdrsCfgPath); err != nil {
+		t.Fatal("Got config error: ", err.Error())
+	}
+}
+
+func TestV2CdrsMysqlInitDataDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitDataDb(cdrsCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// InitDb so we can rely on count
+func TestV2CdrsMysqlInitCdrDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitStorDb(cdrsCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestV2CdrsMysqlInjectUnratedCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var mysqlDb *engine.MySQLStorage
+	if d, err := engine.NewMySQLStorage(cdrsCfg.StorDBHost, cdrsCfg.StorDBPort, cdrsCfg.StorDBName, cdrsCfg.StorDBUser, cdrsCfg.StorDBPass,
+		cdrsCfg.StorDBMaxOpenConns, cdrsCfg.StorDBMaxIdleConns); err != nil {
+		t.Error("Error on opening database connection: ", err)
+		return
+	} else {
+		mysqlDb = d.(*engine.MySQLStorage)
+	}
+	strCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()),
+		TOR: utils.VOICE, AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: utils.META_RATED,
+		Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+		SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC),
+		Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
+		MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201}
+	if err := mysqlDb.SetCdr(strCdr1); err != nil {
+		t.Error(err.Error())
+	}
+}
+
+func TestV2CdrsMysqlStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if _, err := engine.StopStartEngine(cdrsCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestV2CdrsMysqlRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrsRpc, err = jsonrpc.Dial("tcp", cdrsCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+// Insert some CDRs
+func TestV2CdrsMysqlProcessCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans", Rated: true,
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("abcdeftg", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("aererfddf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+	}
+	for _, cdr := range cdrs {
+		if err := cdrsRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil {
+			t.Error("Unexpected error: ", err.Error())
+		} else if reply != utils.OK {
+			t.Error("Unexpected reply received: ", reply)
+		}
+	}
+}
+
+func TestV2CdrsMysqlGetCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply []*engine.ExternalCdr
+	req := utils.RpcCdrsFilter{}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 4 {
+		t.Error("Unexpected number of CDRs returned: ", len(reply))
+	}
+	// CDRs with errors
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0), MaxCost: utils.Float64Pointer(0.0)}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 2 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// CDRs Rated
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 3 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// CDRs non rated OR SkipRated
+	req = utils.RpcCdrsFilter{MaxCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 1 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// Skip Errors
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(0.0), MaxCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 2 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+}
+
+func TestV2CdrsMysqlCountCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply int64
+	req := utils.AttrGetCdrs{}
+	if err := cdrsRpc.Call("ApierV2.CountCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if reply != 4 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+}
+
+// Test Prepaid CDRs without previous costs being calculated
+func TestV2CdrsMysqlProcessPrepaidCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans", Rated: true,
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("abcdeftg2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("aererfddf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+	}
+	tStart := time.Now()
+	for _, cdr := range cdrs {
+		if err := cdrsRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil {
+			t.Error("Unexpected error: ", err.Error())
+		} else if reply != utils.OK {
+			t.Error("Unexpected reply received: ", reply)
+		}
+	}
+	if processDur := time.Now().Sub(tStart); processDur > 1*time.Second {
+		t.Error("Unexpected processing time", processDur)
+	}
+}
+
+func TestV2CdrsMysqlKillEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.KillEngine(*waitRater); err != nil {
+		t.Error(err)
+	}
+}

apier/v2/cdrs_psql_local_test.go

@@ -0,0 +1,248 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"os/exec"
+	"path"
+	"testing"
+	"time"
+)
+
+var cdrsPsqlCfgPath string
+var cdrsPsqlCfg *config.CGRConfig
+var cdrsPsqlRpc *rpc.Client
+var cmdEngineCdrPsql *exec.Cmd
+
+func TestV2CdrsPsqlInitConfig(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrsPsqlCfgPath = path.Join(*dataDir, "conf", "samples", "cdrsv2psql")
+	if cdrsPsqlCfg, err = config.NewCGRConfigFromFolder(cdrsPsqlCfgPath); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestV2CdrsPsqlInitDataDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitDataDb(cdrsPsqlCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// InitDb so we can rely on count
+func TestV2CdrsPsqlInitCdrDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitStorDb(cdrsPsqlCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestV2CdrsPsqlInjectUnratedCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var psqlDb *engine.PostgresStorage
+	if d, err := engine.NewPostgresStorage(cdrsPsqlCfg.StorDBHost, cdrsPsqlCfg.StorDBPort, cdrsPsqlCfg.StorDBName, cdrsPsqlCfg.StorDBUser, cdrsPsqlCfg.StorDBPass,
+		cdrsPsqlCfg.StorDBMaxOpenConns, cdrsPsqlCfg.StorDBMaxIdleConns); err != nil {
+		t.Error("Error on opening database connection: ", err)
+		return
+	} else {
+		psqlDb = d.(*engine.PostgresStorage)
+	}
+	strCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()),
+		TOR: utils.VOICE, AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: utils.META_RATED,
+		Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+		SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC),
+		Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
+		MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201}
+	if err := psqlDb.SetCdr(strCdr1); err != nil {
+		t.Error(err.Error())
+	}
+}
+
+func TestV2CdrsPsqlStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	if cmdEngineCdrPsql, err = engine.StartEngine(cdrsPsqlCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestV2CdrsPsqlPsqlRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrsPsqlRpc, err = jsonrpc.Dial("tcp", cdrsPsqlCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+// Insert some CDRs
+func TestV2CdrsPsqlProcessCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans", Rated: true,
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("abcdeftg", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("aererfddf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+	}
+	for _, cdr := range cdrs {
+		if err := cdrsPsqlRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil {
+			t.Error("Unexpected error: ", err.Error())
+		} else if reply != utils.OK {
+			t.Error("Unexpected reply received: ", reply)
+		}
+	}
+}
+
+func TestV2CdrsPsqlGetCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply []*engine.ExternalCdr
+	req := utils.RpcCdrsFilter{}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 4 {
+		t.Error("Unexpected number of CDRs returned: ", len(reply))
+	}
+	// CDRs with errors
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0), MaxCost: utils.Float64Pointer(0.0)}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 2 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// CDRs Rated
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 3 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// CDRs non rated OR SkipRated
+	req = utils.RpcCdrsFilter{MaxCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 1 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// Skip Errors
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(0.0), MaxCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 2 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+}
+
+func TestV2CdrsPsqlCountCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply int64
+	req := utils.AttrGetCdrs{}
+	if err := cdrsPsqlRpc.Call("ApierV2.CountCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if reply != 4 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+}
+
+// Test Prepaid CDRs without previous costs being calculated
+func TestV2CdrsPsqlProcessPrepaidCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans", Rated: true,
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("abcdeftg2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("aererfddf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+	}
+	tStart := time.Now()
+	for _, cdr := range cdrs {
+		if err := cdrsPsqlRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil {
+			t.Error("Unexpected error: ", err.Error())
+		} else if reply != utils.OK {
+			t.Error("Unexpected reply received: ", reply)
+		}
+	}
+	if processDur := time.Now().Sub(tStart); processDur > 1*time.Second {
+		t.Error("Unexpected processing time", processDur)
+	}
+}
+
+func TestV2CdrsPsqlKillEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.KillEngine(*waitRater); err != nil {
+		t.Error(err)
+	}
+}

apier/v2/tp.go

@@ -0,0 +1,99 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"encoding/base64"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type AttrRemTp struct {
+	TPid string
+}
+
+func (self *ApierV2) RemTP(attrs AttrRemTp, reply *string) error {
+	if len(attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	if err := self.StorDb.RemTpData("", attrs.TPid); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}
+
+func (self *ApierV2) ExportTPToFolder(attrs utils.AttrDirExportTP, exported *utils.ExportedTPStats) error {
+	if len(*attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	dir := self.Config.TpExportPath
+	if attrs.ExportPath != nil {
+		dir = *attrs.ExportPath
+	}
+	fileFormat := utils.CSV
+	if attrs.FileFormat != nil {
+		fileFormat = *attrs.FileFormat
+	}
+	sep := ","
+	if attrs.FieldSeparator != nil {
+		sep = *attrs.FieldSeparator
+	}
+	compress := false
+	if attrs.Compress != nil {
+		compress = *attrs.Compress
+	}
+	tpExporter, err := engine.NewTPExporter(self.StorDb, *attrs.TPid, dir, fileFormat, sep, compress)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if err := tpExporter.Run(); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*exported = *tpExporter.ExportStats()
+	}
+
+	return nil
+}
+
+func (self *ApierV2) ExportTPToZipString(attrs utils.AttrDirExportTP, reply *string) error {
+	if len(*attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	dir := ""
+	fileFormat := utils.CSV
+	if attrs.FileFormat != nil {
+		fileFormat = *attrs.FileFormat
+	}
+	sep := ","
+	if attrs.FieldSeparator != nil {
+		sep = *attrs.FieldSeparator
+	}
+	tpExporter, err := engine.NewTPExporter(self.StorDb, *attrs.TPid, dir, fileFormat, sep, true)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if err := tpExporter.Run(); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = base64.StdEncoding.EncodeToString(tpExporter.GetCacheBuffer().Bytes())
+	return nil
+}

cache2go/cache.go

@@ -2,207 +2,176 @@
 package cache2go
 
 import (
-	"errors"
-	"strings"
 	"sync"
 	"time"
 )
 
-type expiringCacheEntry interface {
-	XCache(key string, expire time.Duration, value expiringCacheEntry)
-	timer() *time.Timer
-	age() time.Duration
-	KeepAlive()
-}
-
-// Structure that must be embeded in the objectst that must be cached with expiration.
-// If the expiration is not needed this can be ignored
-type XEntry struct {
-	sync.Mutex
-	key            string
-	keepAlive      bool
-	expireDuration time.Duration
-	timestamp      time.Time
-	t              *time.Timer
-}
+const (
+	PREFIX_LEN   = 4
+	KIND_ADD     = "ADD"
+	KIND_ADP     = "ADP"
+	KIND_REM     = "REM"
+	KIND_PRF     = "PRF"
+	DOUBLE_CACHE = true
+)
 
 type timestampedValue struct {
 	timestamp time.Time
 	value     interface{}
 }
 
+func (tsv timestampedValue) Value() interface{} {
+	return tsv.value
+}
+
+type transactionItem struct {
+	key   string
+	value interface{}
+	kind  string
+}
+
+func init() {
+	if DOUBLE_CACHE {
+		cache = newDoubleStore()
+	} else {
+		cache = newSimpleStore()
+	}
+}
+
 var (
-	xcache = make(map[string]expiringCacheEntry)
-	xMux   sync.RWMutex
-	cache  = make(map[string]timestampedValue)
-	mux    sync.RWMutex
+	mux   sync.RWMutex
+	cache cacheStore
+	// transaction stuff
+	transactionBuffer []transactionItem
+	transactionMux    sync.Mutex
+	transactionON     = false
+	transactionLock   = false
 )
 
-// The main function to cache with expiration
-func (xe *XEntry) XCache(key string, expire time.Duration, value expiringCacheEntry) {
-	xe.keepAlive = true
-	xe.key = key
-	xe.expireDuration = expire
-	xe.timestamp = time.Now()
-	xMux.Lock()
-	xcache[key] = value
-	xMux.Unlock()
-	go xe.expire()
+func BeginTransaction() {
+	transactionMux.Lock()
+	transactionLock = true
+	transactionON = true
 }
 
-// The internal mechanism for expiartion
-func (xe *XEntry) expire() {
-	for xe.keepAlive {
-		xe.Lock()
-		xe.keepAlive = false
-		xe.Unlock()
-		xe.t = time.NewTimer(xe.expireDuration)
-		<-xe.t.C
-		if !xe.keepAlive {
-			xMux.Lock()
-			delete(xcache, xe.key)
-			xMux.Unlock()
+func RollbackTransaction() {
+	transactionBuffer = nil
+	transactionLock = false
+	transactionON = false
+	transactionMux.Unlock()
+}
+
+func CommitTransaction() {
+	transactionON = false
+	// apply all transactioned items
+	mux.Lock()
+	for _, item := range transactionBuffer {
+		switch item.kind {
+		case KIND_REM:
+			RemKey(item.key)
+		case KIND_PRF:
+			RemPrefixKey(item.key)
+		case KIND_ADD:
+			Cache(item.key, item.value)
+		case KIND_ADP:
+			CachePush(item.key, item.value)
 		}
 	}
-}
-
-// Getter for the timer
-func (xe *XEntry) timer() *time.Timer {
-	return xe.t
-}
-
-func (xe *XEntry) age() time.Duration {
-	return time.Since(xe.timestamp)
-
-}
-
-// Mark entry to be kept another expirationDuration period
-func (xe *XEntry) KeepAlive() {
-	xe.Lock()
-	defer xe.Unlock()
-	xe.keepAlive = true
-}
-
-// Get an entry from the expiration cache and mark it for keeping alive
-func GetXCached(key string) (ece expiringCacheEntry, err error) {
-	xMux.RLock()
-	defer xMux.RUnlock()
-	if r, ok := xcache[key]; ok {
-		r.KeepAlive()
-		return r, nil
-	}
-	return nil, errors.New("not found")
+	mux.Unlock()
+	transactionBuffer = nil
+	transactionLock = false
+	transactionMux.Unlock()
 }
 
 // The function to be used to cache a key/value pair when expiration is not needed
 func Cache(key string, value interface{}) {
-	mux.Lock()
-	defer mux.Unlock()
-	cache[key] = timestampedValue{time.Now(), value}
+	if !transactionLock {
+		mux.Lock()
+		defer mux.Unlock()
+	}
+	if !transactionON {
+		cache.Put(key, value)
+		//fmt.Println("ADD: ", key)
+	} else {
+		transactionBuffer = append(transactionBuffer, transactionItem{key: key, value: value, kind: KIND_ADD})
+	}
+}
+
+// Appends to an existing slice in the cache key
+func CachePush(key string, value interface{}) {
+	if !transactionLock {
+		mux.Lock()
+		defer mux.Unlock()
+	}
+	if !transactionON {
+		cache.Append(key, value)
+	} else {
+		transactionBuffer = append(transactionBuffer, transactionItem{key: key, value: value, kind: KIND_ADP})
+	}
 }
 
 // The function to extract a value for a key that never expire
 func GetCached(key string) (v interface{}, err error) {
 	mux.RLock()
 	defer mux.RUnlock()
-	if r, ok := cache[key]; ok {
-		return r.value, nil
-	}
-	return nil, errors.New("not found")
+	return cache.Get(key)
 }
 
 func GetKeyAge(key string) (time.Duration, error) {
 	mux.RLock()
 	defer mux.RUnlock()
-	if r, ok := cache[key]; ok {
-		return time.Since(r.timestamp), nil
-	}
-	return 0, errors.New("not found")
-}
-
-func GetXKeyAge(key string) (time.Duration, error) {
-	xMux.RLock()
-	defer xMux.RUnlock()
-	if r, ok := xcache[key]; ok {
-		return r.age(), nil
-	}
-	return 0, errors.New("not found")
+	return cache.GetAge(key)
 }
 
 func RemKey(key string) {
-	mux.Lock()
-	defer mux.Unlock()
-	delete(cache, key)
+	if !transactionLock {
+		mux.Lock()
+		defer mux.Unlock()
+	}
+	if !transactionON {
+		cache.Delete(key)
+	} else {
+		transactionBuffer = append(transactionBuffer, transactionItem{key: key, kind: KIND_REM})
+	}
 }
 
 func RemPrefixKey(prefix string) {
-	mux.Lock()
-	defer mux.Unlock()
-	for key, _ := range cache {
-		if strings.HasPrefix(key, prefix) {
-			delete(cache, key)
-		}
+	if !transactionLock {
+		mux.Lock()
+		defer mux.Unlock()
 	}
-}
-
-func XRemKey(key string) {
-	xMux.Lock()
-	defer xMux.Unlock()
-	if r, ok := xcache[key]; ok {
-		if r.timer() != nil {
-			r.timer().Stop()
-		}
+	if !transactionON {
+		cache.DeletePrefix(prefix)
+	} else {
+		transactionBuffer = append(transactionBuffer, transactionItem{key: prefix, kind: KIND_PRF})
 	}
-	delete(xcache, key)
-}
-func XRemPrefixKey(prefix string) {
-	xMux.Lock()
-	defer xMux.Unlock()
-	for key, _ := range xcache {
-		if strings.HasPrefix(key, prefix) {
-			if r, ok := xcache[key]; ok {
-				if r.timer() != nil {
-					r.timer().Stop()
-				}
-			}
-			delete(xcache, key)
-		}
-	}
-}
-
-// Delete all keys from expiraton cache
-func XFlush() {
-	xMux.Lock()
-	defer xMux.Unlock()
-	for _, v := range xcache {
-		if v.timer() != nil {
-			v.timer().Stop()
-		}
-	}
-	xcache = make(map[string]expiringCacheEntry)
 }
 
 // Delete all keys from cache
 func Flush() {
 	mux.Lock()
 	defer mux.Unlock()
-	cache = make(map[string]timestampedValue)
+	if DOUBLE_CACHE {
+		cache = newDoubleStore()
+	} else {
+		cache = newSimpleStore()
+	}
 }
 
 func CountEntries(prefix string) (result int) {
-	for key, _ := range cache {
-		if strings.HasPrefix(key, prefix) {
-			result++
-		}
-	}
-	return
+	mux.RLock()
+	defer mux.RUnlock()
+	return cache.CountEntriesForPrefix(prefix)
 }
 
-func XCountEntries(prefix string) (result int) {
-	for key, _ := range xcache {
-		if strings.HasPrefix(key, prefix) {
-			result++
-		}
-	}
-	return
+func GetAllEntries(prefix string) (map[string]timestampedValue, error) {
+	mux.RLock()
+	defer mux.RUnlock()
+	return cache.GetAllForPrefix(prefix)
+}
+
+func GetEntriesKeys(prefix string) (keys []string) {
+	mux.RLock()
+	defer mux.RUnlock()
+	return cache.GetKeysForPrefix(prefix)
 }

cache2go/cache_test.go

@@ -1,143 +1,106 @@
 package cache2go
 
-import (
-	"testing"
-	"time"
-)
-
-type myStruct struct {
-	XEntry
-	data string
-}
-
-func TestCache(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 1*time.Second, a)
-	b, err := GetXCached("mama")
-	if err != nil || b == nil || b != a {
-		t.Error("Error retriving data from cache", err)
-	}
-}
-
-func TestCacheExpire(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 1*time.Second, a)
-	b, err := GetXCached("mama")
-	if err != nil || b == nil || b.(*myStruct).data != "mama are mere" {
-		t.Error("Error retriving data from cache", err)
-	}
-	time.Sleep(1001 * time.Millisecond)
-	b, err = GetXCached("mama")
-	if err == nil || b != nil {
-		t.Error("Error expiring data")
-	}
-}
-
-func TestCacheKeepAlive(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 1*time.Second, a)
-	b, err := GetXCached("mama")
-	if err != nil || b == nil || b.(*myStruct).data != "mama are mere" {
-		t.Error("Error retriving data from cache", err)
-	}
-	time.Sleep(500 * time.Millisecond)
-	b.KeepAlive()
-	time.Sleep(501 * time.Millisecond)
-	if err != nil {
-		t.Error("Error keeping cached data alive", err)
-	}
-	time.Sleep(1000 * time.Millisecond)
-	b, err = GetXCached("mama")
-	if err == nil || b != nil {
-		t.Error("Error expiring data")
-	}
-}
-
-func TestFlush(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 10*time.Second, a)
-	time.Sleep(1000 * time.Millisecond)
-	XFlush()
-	b, err := GetXCached("mama")
-	if err == nil || b != nil {
-		t.Error("Error expiring data")
-	}
-}
-
-func TestFlushNoTimout(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 10*time.Second, a)
-	XFlush()
-	b, err := GetXCached("mama")
-	if err == nil || b != nil {
-		t.Error("Error expiring data")
-	}
-}
+import "testing"
 
 func TestRemKey(t *testing.T) {
-	Cache("t1", "test")
-	if t1, err := GetCached("t1"); err != nil || t1 != "test" {
-		t.Error("Error setting cache")
+	Cache("t11_mm", "test")
+	if t1, err := GetCached("t11_mm"); err != nil || t1 != "test" {
+		t.Error("Error setting cache: ", err, t1)
 	}
-	RemKey("t1")
-	if t1, err := GetCached("t1"); err == nil || t1 == "test" {
+	RemKey("t11_mm")
+	if t1, err := GetCached("t11_mm"); err == nil || t1 == "test" {
 		t.Error("Error removing cached key")
 	}
 }
 
-func TestXRemKey(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 10*time.Second, a)
-	if t1, err := GetXCached("mama"); err != nil || t1 != a {
-		t.Error("Error setting xcache")
+func TestTransaction(t *testing.T) {
+	BeginTransaction()
+	Cache("t11_mm", "test")
+	if t1, err := GetCached("t11_mm"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
 	}
-	XRemKey("mama")
-	if t1, err := GetXCached("mama"); err == nil || t1 == a {
-		t.Error("Error removing xcached key: ", err, t1)
+	Cache("t12_mm", "test")
+	RemKey("t11_mm")
+	CommitTransaction()
+	if t1, err := GetCached("t12_mm"); err != nil || t1 != "test" {
+		t.Error("Error commiting transaction")
+	}
+	if t1, err := GetCached("t11_mm"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
 	}
 }
 
-/*
-These tests sometimes fails on drone.io
-func TestGetKeyAge(t *testing.T) {
-	Cache("t1", "test")
-	d, err := GetKeyAge("t1")
-	if err != nil || d > time.Millisecond || d < time.Nanosecond {
-		t.Error("Error getting cache key age: ", d)
+func TestTransactionRem(t *testing.T) {
+	BeginTransaction()
+	Cache("t21_mm", "test")
+	Cache("t21_nn", "test")
+	RemPrefixKey("t21_")
+	CommitTransaction()
+	if t1, err := GetCached("t21_mm"); err == nil || t1 == "test" {
+		t.Error("Error commiting transaction")
+	}
+	if t1, err := GetCached("t21_nn"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
 	}
 }
 
-
-func TestXGetKeyAge(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("t1", 10*time.Second, a)
-	d, err := GetXKeyAge("t1")
-	if err != nil || d > time.Millisecond || d < time.Nanosecond {
-		t.Error("Error getting cache key age: ", d)
+func TestTransactionRollback(t *testing.T) {
+	BeginTransaction()
+	Cache("t31_mm", "test")
+	if t1, err := GetCached("t31_mm"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
+	}
+	Cache("t32_mm", "test")
+	RollbackTransaction()
+	if t1, err := GetCached("t32_mm"); err == nil || t1 == "test" {
+		t.Error("Error commiting transaction")
+	}
+	if t1, err := GetCached("t31_mm"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
+	}
+}
+
+func TestTransactionRemBefore(t *testing.T) {
+	BeginTransaction()
+	RemPrefixKey("t41_")
+	Cache("t41_mm", "test")
+	Cache("t41_nn", "test")
+	CommitTransaction()
+	if t1, err := GetCached("t41_mm"); err != nil || t1 != "test" {
+		t.Error("Error commiting transaction")
+	}
+	if t1, err := GetCached("t41_nn"); err != nil || t1 != "test" {
+		t.Error("Error in transaction cache")
 	}
 }
-*/
 
 func TestRemPrefixKey(t *testing.T) {
-	Cache("x_t1", "test")
-	Cache("y_t1", "test")
-	RemPrefixKey("x_")
-	_, errX := GetCached("x_t1")
-	_, errY := GetCached("y_t1")
+	Cache("xxx_t1", "test")
+	Cache("yyy_t1", "test")
+	RemPrefixKey("xxx_")
+	_, errX := GetCached("xxx_t1")
+	_, errY := GetCached("yyy_t1")
 	if errX == nil || errY != nil {
 		t.Error("Error removing prefix: ", errX, errY)
 	}
 }
 
-func TestXRemPrefixKey(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("x_t1", 10*time.Second, a)
-	a.XCache("y_t1", 10*time.Second, a)
-
-	XRemPrefixKey("x_")
-	_, errX := GetXCached("x_t1")
-	_, errY := GetXCached("y_t1")
-	if errX == nil || errY != nil {
-		t.Error("Error removing prefix: ", errX, errY)
+func TestCachePush(t *testing.T) {
+	CachePush("ccc_t1", "1")
+	CachePush("ccc_t1", "2")
+	v, err := GetCached("ccc_t1")
+	if err != nil || len(v.(map[interface{}]struct{})) != 2 {
+		t.Error("Error in cache push: ", v)
+	}
+}
+
+func TestCount(t *testing.T) {
+	Cache("dst_A1", "1")
+	Cache("dst_A2", "2")
+	Cache("rpf_A3", "3")
+	Cache("dst_A4", "4")
+	Cache("dst_A5", "5")
+	if CountEntries("dst_") != 4 {
+		t.Error("Error countiong entries: ", CountEntries("dst_"))
 	}
 }

cache2go/store.go

@@ -0,0 +1,223 @@
+//Simple caching library with expiration capabilities
+package cache2go
+
+import (
+	"strings"
+	"time"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+type cacheStore interface {
+	Put(string, interface{})
+	Append(string, interface{})
+	Get(string) (interface{}, error)
+	GetAge(string) (time.Duration, error)
+	Delete(string)
+	DeletePrefix(string)
+	CountEntriesForPrefix(string) int
+	GetAllForPrefix(string) (map[string]timestampedValue, error)
+	GetKeysForPrefix(string) []string
+}
+
+// easy to be counted exported by prefix
+type cacheDoubleStore map[string]map[string]timestampedValue
+
+func newDoubleStore() cacheDoubleStore {
+	return make(cacheDoubleStore)
+}
+
+func (cs cacheDoubleStore) Put(key string, value interface{}) {
+	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
+	if _, ok := cs[prefix]; !ok {
+		cs[prefix] = make(map[string]timestampedValue)
+	}
+	cs[prefix][key] = timestampedValue{time.Now(), value}
+}
+
+func (cs cacheDoubleStore) Append(key string, value interface{}) {
+	var elements map[interface{}]struct{} // using map for faster check if element is present
+	if v, err := cs.Get(key); err == nil {
+		elements = v.(map[interface{}]struct{})
+	} else {
+		elements = make(map[interface{}]struct{})
+	}
+	elements[value] = struct{}{}
+	cache.Put(key, elements)
+}
+
+func (cs cacheDoubleStore) Get(key string) (interface{}, error) {
+	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
+	if keyMap, ok := cs[prefix]; ok {
+		if ti, exists := keyMap[key]; exists {
+			return ti.value, nil
+		}
+	}
+	return nil, utils.ErrNotFound
+}
+
+func (cs cacheDoubleStore) GetAge(key string) (time.Duration, error) {
+	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
+	if keyMap, ok := cs[prefix]; ok {
+		if ti, exists := keyMap[key]; exists {
+			return time.Since(ti.timestamp), nil
+		}
+	}
+	return -1, utils.ErrNotFound
+}
+
+func (cs cacheDoubleStore) Delete(key string) {
+	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
+	if keyMap, ok := cs[prefix]; ok {
+		delete(keyMap, key)
+	}
+}
+
+func (cs cacheDoubleStore) DeletePrefix(prefix string) {
+	delete(cs, prefix)
+}
+
+func (cs cacheDoubleStore) CountEntriesForPrefix(prefix string) int {
+	if m, ok := cs[prefix]; ok {
+		return len(m)
+	}
+	return 0
+}
+
+func (cs cacheDoubleStore) GetAllForPrefix(prefix string) (map[string]timestampedValue, error) {
+	if keyMap, ok := cs[prefix]; ok {
+		return keyMap, nil
+	}
+	return nil, utils.ErrNotFound
+}
+
+func (cs cacheDoubleStore) GetKeysForPrefix(prefix string) (keys []string) {
+	prefix, key := prefix[:PREFIX_LEN], prefix[PREFIX_LEN:]
+	if keyMap, ok := cs[prefix]; ok {
+		for iterKey := range keyMap {
+			if len(key) > 0 && strings.HasPrefix(iterKey, key) {
+				keys = append(keys, prefix+iterKey)
+			}
+		}
+	}
+	return
+}
+
+// faster to access
+type cacheSimpleStore struct {
+	cache    map[string]timestampedValue
+	counters map[string]int
+}
+
+func newSimpleStore() cacheSimpleStore {
+	return cacheSimpleStore{
+		cache:    make(map[string]timestampedValue),
+		counters: make(map[string]int),
+	}
+}
+
+func (cs cacheSimpleStore) Put(key string, value interface{}) {
+	if _, ok := cs.cache[key]; !ok {
+		// only count if the key is not already there
+		cs.count(key)
+	}
+	cs.cache[key] = timestampedValue{time.Now(), value}
+}
+
+func (cs cacheSimpleStore) Append(key string, value interface{}) {
+	var elements map[interface{}]struct{}
+	if v, err := cs.Get(key); err == nil {
+		elements = v.(map[interface{}]struct{})
+	} else {
+		elements = make(map[interface{}]struct{})
+	}
+	elements[value] = struct{}{}
+	cache.Put(key, elements)
+}
+
+func (cs cacheSimpleStore) Get(key string) (interface{}, error) {
+	if ti, exists := cs.cache[key]; exists {
+		return ti.value, nil
+	}
+	return nil, utils.ErrNotFound
+}
+
+func (cs cacheSimpleStore) GetAge(key string) (time.Duration, error) {
+	if ti, exists := cs.cache[key]; exists {
+		return time.Since(ti.timestamp), nil
+	}
+
+	return -1, utils.ErrNotFound
+}
+
+func (cs cacheSimpleStore) Delete(key string) {
+	if _, ok := cs.cache[key]; ok {
+		delete(cs.cache, key)
+		cs.descount(key)
+	}
+}
+
+func (cs cacheSimpleStore) DeletePrefix(prefix string) {
+	for key, _ := range cs.cache {
+		if strings.HasPrefix(key, prefix) {
+			delete(cs.cache, key)
+			cs.descount(key)
+		}
+	}
+}
+
+// increments the counter for the specified key prefix
+func (cs cacheSimpleStore) count(key string) {
+	if len(key) < PREFIX_LEN {
+		return
+	}
+	prefix := key[:PREFIX_LEN]
+	if _, ok := cs.counters[prefix]; ok {
+		// increase the value
+		cs.counters[prefix] += 1
+	} else {
+		cs.counters[prefix] = 1
+	}
+}
+
+// decrements the counter for the specified key prefix
+func (cs cacheSimpleStore) descount(key string) {
+	if len(key) < PREFIX_LEN {
+		return
+	}
+	prefix := key[:PREFIX_LEN]
+	if value, ok := cs.counters[prefix]; ok && value > 0 {
+		cs.counters[prefix] -= 1
+	}
+}
+
+func (cs cacheSimpleStore) CountEntriesForPrefix(prefix string) int {
+	if _, ok := cs.counters[prefix]; ok {
+		return cs.counters[prefix]
+	}
+	return 0
+}
+
+func (cs cacheSimpleStore) GetAllForPrefix(prefix string) (map[string]timestampedValue, error) {
+	result := make(map[string]timestampedValue)
+	found := false
+	for key, ti := range cs.cache {
+		if strings.HasPrefix(key, prefix) {
+			result[key[PREFIX_LEN:]] = ti
+			found = true
+		}
+	}
+	if !found {
+		return nil, utils.ErrNotFound
+	}
+	return result, nil
+}
+
+func (cs cacheSimpleStore) GetKeysForPrefix(prefix string) (keys []string) {
+	for key, _ := range cs.cache {
+		if strings.HasPrefix(key, prefix) {
+			keys = append(keys, key)
+		}
+	}
+	return
+}

calls_test.sh

@@ -0,0 +1,9 @@
+#! /usr/bin/env sh
+
+./local_test.sh
+lcl=$?
+echo 'go test github.com/cgrates/cgrates/general_tests -calls'
+go test github.com/cgrates/cgrates/general_tests -calls
+gnr=$?
+
+exit $gen && $gnr

cdrc/cdrc.go

@@ -1,6 +1,6 @@
 /*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -21,173 +21,168 @@ package cdrc
 import (
 	"bufio"
 	"encoding/csv"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
-	"strconv"
-	"strings"
 	"time"
 
-	"github.com/cgrates/cgrates/cdrs"
 	"github.com/cgrates/cgrates/config"
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
-	"github.com/howeyc/fsnotify"
+	"gopkg.in/fsnotify.v1"
 )
 
 const (
-	CSV    = "csv"
-	FS_CSV = "freeswitch_csv"
+	CSV             = "csv"
+	FS_CSV          = "freeswitch_csv"
+	UNPAIRED_SUFFIX = ".unpaired"
 )
 
-func NewCdrc(config *config.CGRConfig, cdrServer *cdrs.CDRS) (*Cdrc, error) {
-	cdrc := &Cdrc{cgrCfg: config, cdrServer: cdrServer}
-	// Before processing, make sure in and out folders exist
-	for _, dir := range []string{cdrc.cgrCfg.CdrcCdrInDir, cdrc.cgrCfg.CdrcCdrOutDir} {
-		if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
-			return nil, fmt.Errorf("Folder %s does not exist", dir)
+// Populates the
+func populateStoredCdrField(cdr *engine.StoredCdr, fieldId, fieldVal string) error {
+	var err error
+	switch fieldId {
+	case utils.TOR:
+		cdr.TOR += fieldVal
+	case utils.ACCID:
+		cdr.AccId += fieldVal
+	case utils.REQTYPE:
+		cdr.ReqType += fieldVal
+	case utils.DIRECTION:
+		cdr.Direction += fieldVal
+	case utils.TENANT:
+		cdr.Tenant += fieldVal
+	case utils.CATEGORY:
+		cdr.Category += fieldVal
+	case utils.ACCOUNT:
+		cdr.Account += fieldVal
+	case utils.SUBJECT:
+		cdr.Subject += fieldVal
+	case utils.DESTINATION:
+		cdr.Destination += fieldVal
+	case utils.SETUP_TIME:
+		if cdr.SetupTime, err = utils.ParseTimeDetectLayout(fieldVal); err != nil {
+			return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
 		}
+	case utils.PDD:
+		if cdr.Pdd, err = utils.ParseDurationWithSecs(fieldVal); err != nil {
+			return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
+		}
+	case utils.ANSWER_TIME:
+		if cdr.AnswerTime, err = utils.ParseTimeDetectLayout(fieldVal); err != nil {
+			return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
+		}
+	case utils.USAGE:
+		if cdr.Usage, err = utils.ParseDurationWithSecs(fieldVal); err != nil {
+			return fmt.Errorf("Cannot parse duration field with value: %s, err: %s", fieldVal, err.Error())
+		}
+	case utils.SUPPLIER:
+		cdr.Supplier += fieldVal
+	case utils.DISCONNECT_CAUSE:
+		cdr.DisconnectCause += fieldVal
+	default: // Extra fields will not match predefined so they all show up here
+		cdr.ExtraFields[fieldId] += fieldVal
 	}
-	if err := cdrc.parseFieldsConfig(); err != nil {
-		return nil, err
+	return nil
+}
+
+// Understands and processes a specific format of cdr (eg: .csv or .fwv)
+type RecordsProcessor interface {
+	ProcessNextRecord() ([]*engine.StoredCdr, error) // Process a single record in the CDR file, return a slice of CDRs since based on configuration we can have more templates
+}
+
+/*
+One instance  of CDRC will act on one folder.
+Common parameters within configs processed:
+ * cdrS, cdrFormat, cdrInDir, cdrOutDir, runDelay
+Parameters specific per config instance:
+ * duMultiplyFactor, cdrSourceId, cdrFilter, cdrFields
+*/
+func NewCdrc(cdrcCfgs map[string]*config.CdrcConfig, httpSkipTlsCheck bool, cdrs engine.Connector, exitChan chan struct{}) (*Cdrc, error) {
+	var cdrcCfg *config.CdrcConfig
+	for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one
+		break
+	}
+	cdrc := &Cdrc{cdrFormat: cdrcCfg.CdrFormat, cdrInDir: cdrcCfg.CdrInDir, cdrOutDir: cdrcCfg.CdrOutDir,
+		runDelay: cdrcCfg.RunDelay, csvSep: cdrcCfg.FieldSeparator,
+		httpSkipTlsCheck: httpSkipTlsCheck, cdrcCfgs: cdrcCfgs, dfltCdrcCfg: cdrcCfg, cdrs: cdrs, exitChan: exitChan, maxOpenFiles: make(chan struct{}, cdrcCfg.MaxOpenFiles),
+	}
+	var processFile struct{}
+	for i := 0; i < cdrcCfg.MaxOpenFiles; i++ {
+		cdrc.maxOpenFiles <- processFile // Empty initiate so we do not need to wait later when we pop
+	}
+	cdrc.cdrSourceIds = make([]string, len(cdrcCfgs))
+	cdrc.duMultiplyFactors = make([]float64, len(cdrcCfgs))
+	cdrc.cdrFilters = make([]utils.RSRFields, len(cdrcCfgs))
+	cdrc.cdrFields = make([][]*config.CfgCdrField, len(cdrcCfgs))
+	idx := 0
+	var err error
+	for _, cfg := range cdrcCfgs {
+		if idx == 0 { // Steal the config from just one instance since it should be the same for all
+			cdrc.failedCallsPrefix = cfg.FailedCallsPrefix
+			if cdrc.partialRecordsCache, err = NewPartialRecordsCache(cdrcCfg.PartialRecordCache, cdrcCfg.CdrOutDir, cdrcCfg.FieldSeparator); err != nil {
+				return nil, err
+			}
+		}
+		cdrc.cdrSourceIds[idx] = cfg.CdrSourceId
+		cdrc.duMultiplyFactors[idx] = cfg.DataUsageMultiplyFactor
+		cdrc.cdrFilters[idx] = cfg.CdrFilter
+		cdrc.cdrFields[idx] = cfg.ContentFields
+		idx += 1
+	}
+	// Before processing, make sure in and out folders exist
+	for _, dir := range []string{cdrc.cdrInDir, cdrc.cdrOutDir} {
+		if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
+			return nil, fmt.Errorf("Nonexistent folder: %s", dir)
+		}
 	}
 	cdrc.httpClient = new(http.Client)
 	return cdrc, nil
 }
 
 type Cdrc struct {
-	cgrCfg       *config.CGRConfig
-	cdrServer    *cdrs.CDRS
-	cfgCdrFields map[string]string // Key is the name of the field
-	httpClient   *http.Client
+	cdrFormat,
+	cdrInDir,
+	cdrOutDir string
+	failedCallsPrefix   string   // Configured failedCallsPrefix, used in case of flatstore CDRs
+	cdrSourceIds        []string // Should be in sync with cdrFields on indexes
+	runDelay            time.Duration
+	csvSep              rune
+	duMultiplyFactors   []float64
+	cdrFilters          []utils.RSRFields       // Should be in sync with cdrFields on indexes
+	cdrFields           [][]*config.CfgCdrField // Profiles directly connected with cdrFilters
+	httpSkipTlsCheck    bool
+	cdrcCfgs            map[string]*config.CdrcConfig // All cdrc config profiles attached to this CDRC (key will be profile instance name)
+	dfltCdrcCfg         *config.CdrcConfig
+	cdrs                engine.Connector
+	httpClient          *http.Client
+	exitChan            chan struct{}
+	maxOpenFiles        chan struct{}        // Maximum number of simultaneous files processed
+	partialRecordsCache *PartialRecordsCache // Shared between all files in the folder we process
 }
 
 // When called fires up folder monitoring, either automated via inotify or manual by sleeping between processing
 func (self *Cdrc) Run() error {
-	if self.cgrCfg.CdrcRunDelay == time.Duration(0) { // Automated via inotify
+	if self.runDelay == time.Duration(0) { // Automated via inotify
 		return self.trackCDRFiles()
 	}
-	// No automated, process and sleep approach
+	// Not automated, process and sleep approach
 	for {
+		select {
+		case exitChan := <-self.exitChan: // Exit, reinject exitChan for other CDRCs
+			self.exitChan <- exitChan
+			engine.Logger.Info(fmt.Sprintf("<Cdrc> Shutting down CDRC on path %s.", self.cdrInDir))
+			return nil
+		default:
+		}
 		self.processCdrDir()
-		time.Sleep(self.cgrCfg.CdrcRunDelay)
+		time.Sleep(self.runDelay)
 	}
 }
 
-// Loads all fields (primary and extra) into cfgCdrFields, do some pre-checks (eg: in case of csv make sure that values are integers)
-func (self *Cdrc) parseFieldsConfig() error {
-	var err error
-	self.cfgCdrFields = map[string]string{
-		utils.ACCID:       self.cgrCfg.CdrcAccIdField,
-		utils.REQTYPE:     self.cgrCfg.CdrcReqTypeField,
-		utils.DIRECTION:   self.cgrCfg.CdrcDirectionField,
-		utils.TENANT:      self.cgrCfg.CdrcTenantField,
-		utils.TOR:         self.cgrCfg.CdrcTorField,
-		utils.ACCOUNT:     self.cgrCfg.CdrcAccountField,
-		utils.SUBJECT:     self.cgrCfg.CdrcSubjectField,
-		utils.DESTINATION: self.cgrCfg.CdrcDestinationField,
-		utils.SETUP_TIME:  self.cgrCfg.CdrcSetupTimeField,
-		utils.ANSWER_TIME: self.cgrCfg.CdrcAnswerTimeField,
-		utils.DURATION:    self.cgrCfg.CdrcDurationField,
-	}
-
-	// Add extra fields here, config extra fields in the form of []string{"fieldName1:indxInCsv1","fieldName2: indexInCsv2"}
-	for _, fieldWithIdx := range self.cgrCfg.CdrcExtraFields {
-		splt := strings.Split(fieldWithIdx, ":")
-		if len(splt) != 2 {
-			return errors.New("Cannot parse cdrc.extra_fields")
-		}
-		if utils.IsSliceMember(utils.PrimaryCdrFields, splt[0]) {
-			return errors.New("Extra cdrc.extra_fields overwriting primary fields")
-		}
-		self.cfgCdrFields[splt[0]] = splt[1]
-	}
-	// Fields populated, do some sanity checks here
-	for cdrField, cfgVal := range self.cfgCdrFields {
-		if utils.IsSliceMember([]string{CSV, FS_CSV}, self.cgrCfg.CdrcCdrType) && !strings.HasPrefix(cfgVal, utils.STATIC_VALUE_PREFIX) {
-			if _, err = strconv.Atoi(cfgVal); err != nil {
-				return fmt.Errorf("Cannot parse configuration field %s into integer", cdrField)
-			}
-		}
-	}
-	return nil
-}
-
-// Takes the record out of csv and turns it into http form which can be posted
-func (self *Cdrc) recordAsStoredCdr(record []string) (*utils.StoredCdr, error) {
-	ratedCdr := &utils.StoredCdr{CdrSource: self.cgrCfg.CdrcSourceId, ExtraFields: map[string]string{}, Cost: -1}
-	var err error
-	for cfgFieldName, cfgFieldVal := range self.cfgCdrFields {
-		var fieldVal string
-		if strings.HasPrefix(cfgFieldVal, utils.STATIC_VALUE_PREFIX) {
-			fieldVal = cfgFieldVal[1:]
-		} else if utils.IsSliceMember([]string{CSV, FS_CSV}, self.cgrCfg.CdrcCdrType) {
-			if cfgFieldIdx, err := strconv.Atoi(cfgFieldVal); err != nil { // Should in theory never happen since we have already parsed config
-				return nil, err
-			} else if len(record) <= cfgFieldIdx {
-				return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cfgFieldName)
-			} else {
-				fieldVal = record[cfgFieldIdx]
-			}
-		} else { // Modify here when we add more supported cdr formats
-			fieldVal = "UNKNOWN"
-		}
-		switch cfgFieldName {
-		case utils.ACCID:
-			ratedCdr.CgrId = utils.FSCgrId(fieldVal)
-			ratedCdr.AccId = fieldVal
-		case utils.REQTYPE:
-			ratedCdr.ReqType = fieldVal
-		case utils.DIRECTION:
-			ratedCdr.Direction = fieldVal
-		case utils.TENANT:
-			ratedCdr.Tenant = fieldVal
-		case utils.TOR:
-			ratedCdr.TOR = fieldVal
-		case utils.ACCOUNT:
-			ratedCdr.Account = fieldVal
-		case utils.SUBJECT:
-			ratedCdr.Subject = fieldVal
-		case utils.DESTINATION:
-			ratedCdr.Destination = fieldVal
-		case utils.SETUP_TIME:
-			if ratedCdr.SetupTime, err = utils.ParseTimeDetectLayout(fieldVal); err != nil {
-				return nil, fmt.Errorf("Cannot parse answer time field, err: %s", err.Error())
-			}
-		case utils.ANSWER_TIME:
-			if ratedCdr.AnswerTime, err = utils.ParseTimeDetectLayout(fieldVal); err != nil {
-				return nil, fmt.Errorf("Cannot parse answer time field, err: %s", err.Error())
-			}
-		case utils.DURATION:
-			if ratedCdr.Duration, err = utils.ParseDurationWithSecs(fieldVal); err != nil {
-				return nil, fmt.Errorf("Cannot parse duration field, err: %s", err.Error())
-			}
-		default: // Extra fields will not match predefined so they all show up here
-			ratedCdr.ExtraFields[cfgFieldName] = fieldVal
-		}
-
-	}
-	return ratedCdr, nil
-}
-
-// One run over the CDR folder
-func (self *Cdrc) processCdrDir() error {
-	engine.Logger.Info(fmt.Sprintf("<Cdrc> Parsing folder %s for CDR files.", self.cgrCfg.CdrcCdrInDir))
-	filesInDir, _ := ioutil.ReadDir(self.cgrCfg.CdrcCdrInDir)
-	for _, file := range filesInDir {
-		if self.cgrCfg.CdrcCdrType != FS_CSV || path.Ext(file.Name()) != ".csv" {
-			if err := self.processFile(path.Join(self.cgrCfg.CdrcCdrInDir, file.Name())); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
 // Watch the specified folder for file moves and parse the files on events
 func (self *Cdrc) trackCDRFiles() (err error) {
 	watcher, err := fsnotify.NewWatcher()
@@ -195,27 +190,53 @@ func (self *Cdrc) trackCDRFiles() (err error) {
 		return
 	}
 	defer watcher.Close()
-	err = watcher.Watch(self.cgrCfg.CdrcCdrInDir)
+	err = watcher.Add(self.cdrInDir)
 	if err != nil {
 		return
 	}
-	engine.Logger.Info(fmt.Sprintf("<Cdrc> Monitoring %s for file moves.", self.cgrCfg.CdrcCdrInDir))
+	engine.Logger.Info(fmt.Sprintf("<Cdrc> Monitoring %s for file moves.", self.cdrInDir))
 	for {
 		select {
-		case ev := <-watcher.Event:
-			if ev.IsCreate() && (self.cgrCfg.CdrcCdrType != FS_CSV || path.Ext(ev.Name) != ".csv") {
-				if err = self.processFile(ev.Name); err != nil {
-					engine.Logger.Err(fmt.Sprintf("Processing file %s, error: %s", ev.Name, err.Error()))
-				}
+		case exitChan := <-self.exitChan: // Exit, reinject exitChan for other CDRCs
+			self.exitChan <- exitChan
+			engine.Logger.Info(fmt.Sprintf("<Cdrc> Shutting down CDRC on path %s.", self.cdrInDir))
+			return nil
+		case ev := <-watcher.Events:
+			if ev.Op&fsnotify.Create == fsnotify.Create && (self.cdrFormat != FS_CSV || path.Ext(ev.Name) != ".csv") {
+				go func() { //Enable async processing here
+					if err = self.processFile(ev.Name); err != nil {
+						engine.Logger.Err(fmt.Sprintf("Processing file %s, error: %s", ev.Name, err.Error()))
+					}
+				}()
 			}
-		case err := <-watcher.Error:
+		case err := <-watcher.Errors:
 			engine.Logger.Err(fmt.Sprintf("Inotify error: %s", err.Error()))
 		}
 	}
 }
 
+// One run over the CDR folder
+func (self *Cdrc) processCdrDir() error {
+	engine.Logger.Info(fmt.Sprintf("<Cdrc> Parsing folder %s for CDR files.", self.cdrInDir))
+	filesInDir, _ := ioutil.ReadDir(self.cdrInDir)
+	for _, file := range filesInDir {
+		if self.cdrFormat != FS_CSV || path.Ext(file.Name()) != ".csv" {
+			go func() { //Enable async processing here
+				if err := self.processFile(path.Join(self.cdrInDir, file.Name())); err != nil {
+					engine.Logger.Err(fmt.Sprintf("Processing file %s, error: %s", file, err.Error()))
+				}
+			}()
+		}
+	}
+	return nil
+}
+
 // Processe file at filePath and posts the valid cdr rows out of it
 func (self *Cdrc) processFile(filePath string) error {
+	if cap(self.maxOpenFiles) != 0 { // 0 goes for no limit
+		processFile := <-self.maxOpenFiles // Queue here for maxOpenFiles
+		defer func() { self.maxOpenFiles <- processFile }()
+	}
 	_, fn := path.Split(filePath)
 	engine.Logger.Info(fmt.Sprintf("<Cdrc> Parsing: %s", filePath))
 	file, err := os.Open(filePath)
@@ -224,38 +245,53 @@ func (self *Cdrc) processFile(filePath string) error {
 		engine.Logger.Crit(err.Error())
 		return err
 	}
-	csvReader := csv.NewReader(bufio.NewReader(file))
+	var recordsProcessor RecordsProcessor
+	switch self.cdrFormat {
+	case CSV, FS_CSV, utils.KAM_FLATSTORE, utils.OSIPS_FLATSTORE:
+		csvReader := csv.NewReader(bufio.NewReader(file))
+		csvReader.Comma = self.csvSep
+		recordsProcessor = NewCsvRecordsProcessor(csvReader, self.cdrFormat, fn, self.failedCallsPrefix,
+			self.cdrSourceIds, self.duMultiplyFactors, self.cdrFilters, self.cdrFields, self.httpSkipTlsCheck, self.partialRecordsCache)
+	case utils.FWV:
+		recordsProcessor = NewFwvRecordsProcessor(file, self.cdrcCfgs, self.dfltCdrcCfg, self.httpClient, self.httpSkipTlsCheck)
+	default:
+		return fmt.Errorf("Unsupported CDR format: %s", self.cdrFormat)
+	}
+	procRowNr := 0
+	cdrsPosted := 0
+	timeStart := time.Now()
 	for {
-		record, err := csvReader.Read()
+		cdrs, err := recordsProcessor.ProcessNextRecord()
 		if err != nil && err == io.EOF {
-			break // End of file
-		} else if err != nil {
-			engine.Logger.Err(fmt.Sprintf("<Cdrc> Error in csv file: %s", err.Error()))
-			continue // Other csv related errors, ignore
+			break
 		}
-		rawCdr, err := self.recordAsStoredCdr(record)
+		procRowNr += 1
 		if err != nil {
-			engine.Logger.Err(fmt.Sprintf("<Cdrc> Error in csv file: %s", err.Error()))
+			engine.Logger.Err(fmt.Sprintf("<Cdrc> Row %d, error: %s", procRowNr, err.Error()))
 			continue
 		}
-		if self.cgrCfg.CdrcCdrs == utils.INTERNAL {
-			if err := self.cdrServer.ProcessRawCdr(rawCdr); err != nil {
-				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed posting CDR, error: %s", err.Error()))
+		for _, storedCdr := range cdrs { // Send CDRs to CDRS
+			var reply string
+			if self.dfltCdrcCfg.DryRun {
+				engine.Logger.Info(fmt.Sprintf("<Cdrc> DryRun CDR: %+v", storedCdr))
 				continue
 			}
-		} else { // CDRs listening on IP
-			if _, err := self.httpClient.PostForm(fmt.Sprintf("http://%s/cgr", self.cgrCfg.HTTPListen), rawCdr.AsRawCdrHttpForm()); err != nil {
-				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed posting CDR, error: %s", err.Error()))
-				continue
+			if err := self.cdrs.ProcessCdr(storedCdr, &reply); err != nil {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed sending CDR, %+v, error: %s", storedCdr, err.Error()))
+			} else if reply != "OK" {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Received unexpected reply for CDR, %+v, reply: %s", storedCdr, reply))
+			} else {
+				cdrsPosted += 1
 			}
 		}
 	}
 	// Finished with file, move it to processed folder
-	newPath := path.Join(self.cgrCfg.CdrcCdrOutDir, fn)
+	newPath := path.Join(self.cdrOutDir, fn)
 	if err := os.Rename(filePath, newPath); err != nil {
 		engine.Logger.Err(err.Error())
 		return err
 	}
-	engine.Logger.Info(fmt.Sprintf("Finished processing %s, moved to %s", fn, newPath))
+	engine.Logger.Info(fmt.Sprintf("Finished processing %s, moved to %s. Total records processed: %d, CDRs posted: %d, run duration: %s",
+		fn, newPath, procRowNr, cdrsPosted, time.Now().Sub(timeStart)))
 	return nil
 }

cdrc/cdrc_local_test.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -22,15 +22,16 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
 	"io/ioutil"
-	"os/exec"
 	"os"
+	"os/exec"
 	"path"
 	"testing"
 	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
 )
 
 /*
@@ -48,17 +49,14 @@ README:
 
 var cfgPath string
 var cfg *config.CGRConfig
+var cdrcCfgs map[string]*config.CdrcConfig
+var cdrcCfg *config.CdrcConfig
 
 var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args
 var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here")
 var storDbType = flag.String("stordb_type", "mysql", "The type of the storDb database <mysql>")
 var waitRater = flag.Int("wait_rater", 300, "Number of miliseconds to wait for rater to start and cache")
 
-func init() {
-	cfgPath = path.Join(*dataDir, "conf", "samples", "apier_local_test.cfg")
-	cfg, _ = config.NewCGRConfig(&cfgPath)
-}
-
 var fileContent1 = `accid11,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1
 accid12,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1
 dummy_data
@@ -70,13 +68,18 @@ accid22,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:0
 #accid1,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1
 accid23,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1`
 
+var fileContent3 = `accid31;prepaid;out;cgrates.org;call;1001;1001;+4986517174963;2013-02-03 19:54:00;62;supplier1;172.16.1.1
+accid32;prepaid;out;cgrates.org;call;1001;1001;+4986517174963;2013-02-03 19:54:00;62;supplier1;172.16.1.1
+#accid1;prepaid;out;cgrates.org;call;1001;1001;+4986517174963;2013-02-03 19:54:00;62;supplier1;172.16.1.1
+accid33;prepaid;out;cgrates.org;call;1001;1001;+4986517174963;2013-02-03 19:54:00;62;supplier1;172.16.1.1`
+
 func startEngine() error {
 	enginePath, err := exec.LookPath("cgr-engine")
 	if err != nil {
 		return errors.New("Cannot find cgr-engine executable")
 	}
 	stopEngine()
-	engine := exec.Command(enginePath, "-cdrs", "-config", cfgPath)
+	engine := exec.Command(enginePath, "-config", cfgPath)
 	if err := engine.Start(); err != nil {
 		return fmt.Errorf("Cannot start cgr-engine: %s", err.Error())
 	}
@@ -89,6 +92,18 @@ func stopEngine() error {
 	return nil
 }
 
+// Need it here and not in init since Travis has no possibility to load local file
+func TestLoadConfigt(*testing.T) {
+	if !*testLocal {
+		return
+	}
+	cfgPath = path.Join(*dataDir, "conf", "samples", "apier")
+	cfg, _ = config.NewCGRConfigFromFolder(cfgPath)
+	if len(cfg.CdrcProfiles) > 0 {
+		cdrcCfgs = cfg.CdrcProfiles["/var/log/cgrates/cdrc/in"]
+	}
+}
+
 func TestEmptyTables(t *testing.T) {
 	if !*testLocal {
 		return
@@ -97,12 +112,12 @@ func TestEmptyTables(t *testing.T) {
 		t.Fatal("Unsupported storDbType")
 	}
 	var mysql *engine.MySQLStorage
-	if d, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass); err != nil {
+	if d, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns); err != nil {
 		t.Fatal("Error on opening database connection: ", err)
 	} else {
 		mysql = d.(*engine.MySQLStorage)
 	}
-	for _, scriptName := range []string{engine.CREATE_CDRS_TABLES_SQL, engine.CREATE_COSTDETAILS_TABLES_SQL, engine.CREATE_MEDIATOR_TABLES_SQL, engine.CREATE_TARIFFPLAN_TABLES_SQL} {
+	for _, scriptName := range []string{utils.CREATE_CDRS_TABLES_SQL, utils.CREATE_TARIFFPLAN_TABLES_SQL} {
 		if err := mysql.CreateTablesFromScript(path.Join(*dataDir, "storage", *storDbType, scriptName)); err != nil {
 			t.Fatal("Error on mysql creation: ", err.Error())
 			return // No point in going further
@@ -120,31 +135,84 @@ func TestCreateCdrFiles(t *testing.T) {
 	if !*testLocal {
 		return
 	}
-	if err := os.RemoveAll(cfg.CdrcCdrInDir); err != nil {
-		t.Fatal("Error removing folder: ", cfg.CdrcCdrInDir, err)
+	if cdrcCfgs == nil {
+		t.Fatal("Empty default cdrc configuration")
 	}
-	if err := os.MkdirAll(cfg.CdrcCdrInDir, 0755); err != nil {
-		t.Fatal("Error creating folder: ", cfg.CdrcCdrInDir, err)
+	for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one
+		break
 	}
-	if err := ioutil.WriteFile(path.Join(cfg.CdrcCdrInDir, "file1.csv"), []byte(fileContent1), 0644); err != nil {
+	if err := os.RemoveAll(cdrcCfg.CdrInDir); err != nil {
+		t.Fatal("Error removing folder: ", cdrcCfg.CdrInDir, err)
+	}
+	if err := os.MkdirAll(cdrcCfg.CdrInDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", cdrcCfg.CdrInDir, err)
+	}
+	if err := os.RemoveAll(cdrcCfg.CdrOutDir); err != nil {
+		t.Fatal("Error removing folder: ", cdrcCfg.CdrOutDir, err)
+	}
+	if err := os.MkdirAll(cdrcCfg.CdrOutDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", cdrcCfg.CdrOutDir, err)
+	}
+	if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file1.csv"), []byte(fileContent1), 0644); err != nil {
 		t.Fatal(err.Error)
 	}
-	if err := ioutil.WriteFile(path.Join(cfg.CdrcCdrInDir, "file2.csv"), []byte(fileContent2), 0644); err != nil {
+	if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file2.csv"), []byte(fileContent2), 0644); err != nil {
 		t.Fatal(err.Error)
 	}
+
 }
 
 func TestProcessCdrDir(t *testing.T) {
 	if !*testLocal {
 		return
 	}
-	if cfg.CdrcCdrs == utils.INTERNAL { // For now we only test over network
-		return
+	var cdrcCfg *config.CdrcConfig
+	for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one
+		break
+	}
+	if cdrcCfg.Cdrs == utils.INTERNAL { // For now we only test over network
+		cdrcCfg.Cdrs = "127.0.0.1:2013"
 	}
 	if err := startEngine(); err != nil {
 		t.Fatal(err.Error())
 	}
-	cdrc, err := NewCdrc(cfg, nil)
+	cdrc, err := NewCdrc(cdrcCfgs, true, nil, make(chan struct{}))
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	if err := cdrc.processCdrDir(); err != nil {
+		t.Error(err)
+	}
+	stopEngine()
+}
+
+// Creates cdr files and starts the engine
+func TestCreateCdr3File(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := os.RemoveAll(cdrcCfg.CdrInDir); err != nil {
+		t.Fatal("Error removing folder: ", cdrcCfg.CdrInDir, err)
+	}
+	if err := os.MkdirAll(cdrcCfg.CdrInDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", cdrcCfg.CdrInDir, err)
+	}
+	if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file3.csv"), []byte(fileContent3), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+}
+
+func TestProcessCdr3Dir(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if cdrcCfg.Cdrs == utils.INTERNAL { // For now we only test over network
+		cdrcCfg.Cdrs = "127.0.0.1:2013"
+	}
+	if err := startEngine(); err != nil {
+		t.Fatal(err.Error())
+	}
+	cdrc, err := NewCdrc(cdrcCfgs, true, nil, make(chan struct{}))
 	if err != nil {
 		t.Fatal(err.Error())
 	}

cdrc/cdrc_test.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -18,91 +18,301 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 
 package cdrc
 
-import (
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/utils"
-	"reflect"
-	"testing"
-	"time"
-)
-
-func TestParseFieldsConfig(t *testing.T) {
-	// Test default config
-	cgrConfig, _ := config.NewDefaultCGRConfig()
-	// Test primary field index definition
-	cgrConfig.CdrcAccIdField = "detect_me"
-	cdrc := &Cdrc{cgrCfg: cgrConfig}
-	if err := cdrc.parseFieldsConfig(); err == nil {
-		t.Error("Failed detecting error in accounting id definition", err)
+/*
+func TestNewPartialFlatstoreRecord(t *testing.T) {
+	ePr := &PartialFlatstoreRecord{Method: "INVITE", AccId: "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:02daec40c548625ac", Timestamp: time.Date(2015, 7, 9, 15, 6, 48, 0, time.UTC),
+		Values: []string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454408", "*prepaid", "1001", "1002", "", "3401:2069362475"}}
+	if pr, err := NewPartialFlatstoreRecord(ePr.Values); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(ePr, pr) {
+		t.Errorf("Expecting: %+v, received: %+v", ePr, pr)
 	}
-	cgrConfig.CdrcAccIdField = "^static_val"
-	cgrConfig.CdrcSubjectField = "1"
-	cdrc = &Cdrc{cgrCfg: cgrConfig}
-	if err := cdrc.parseFieldsConfig(); err != nil {
-		t.Error("Failed to corectly parse primary fields %v", cdrc.cfgCdrFields)
-	}
-	cgrConfig.CdrcExtraFields = []string{"^static_val:orig_ip"}
-	// Test extra field index definition
-	cgrConfig.CdrcAccIdField = "0" // Put back as int
-	cgrConfig.CdrcExtraFields = []string{"supplier1", "orig_ip:11"}
-	cdrc = &Cdrc{cgrCfg: cgrConfig}
-	if err := cdrc.parseFieldsConfig(); err == nil {
-		t.Error("Failed detecting error in extra fields definition", err)
-	}
-	cgrConfig.CdrcExtraFields = []string{"supplier1:^top_supplier", "orig_ip:11"}
-	cdrc = &Cdrc{cgrCfg: cgrConfig}
-	if err := cdrc.parseFieldsConfig(); err != nil {
-		t.Errorf("Failed to corectly parse extra fields %v", cdrc.cfgCdrFields)
+	if _, err := NewPartialFlatstoreRecord([]string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK"}); err == nil || err.Error() != "MISSING_IE" {
+		t.Error(err)
 	}
 }
+*/
 
-func TestRecordAsStoredCdr(t *testing.T) {
-	cgrConfig, _ := config.NewDefaultCGRConfig()
-	cgrConfig.CdrcExtraFields = []string{"supplier:11"}
-	cdrc := &Cdrc{cgrCfg: cgrConfig}
-	if err := cdrc.parseFieldsConfig(); err != nil {
-		t.Error("Failed parsing default fieldIndexesFromConfig", err)
+/*
+func TestOsipsFlatstoreCdrs(t *testing.T) {
+	flatstoreCdrs := `
+INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475
+BYE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454410|||||3401:2069362475
+INVITE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454647|*postpaid|1002|1001||1877:893549741
+BYE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454651|||||1877:893549741
+INVITE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454657|*prepaid|1001|1002||2407:1884881533
+BYE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454661|||||2407:1884881533
+INVITE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454690|*prepaid|1001|1002||3099:1909036290
+BYE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454692|||||3099:1909036290
+`
+
+	eCdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{
+			CgrId:           "e61034c34148a7c4f40623e00ca5e551d1408bf3",
+			TOR:             utils.VOICE,
+			AccId:           "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:02daec40c548625ac",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 06, 48, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 06, 48, 0, time.UTC),
+			Usage:           time.Duration(2) * time.Second,
+			DisconnectCause: "200 OK",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "3401:2069362475",
+			},
+			Cost: -1,
+		},
+		&engine.StoredCdr{
+			CgrId:           "3ed64a28190e20ac8a6fd8fd48cb23efbfeb7a17",
+			TOR:             utils.VOICE,
+			AccId:           "214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0f9d3d5c3c863a6e3",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_POSTPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1002",
+			Subject:         "1002",
+			Destination:     "1001",
+			SetupTime:       time.Date(2015, 7, 9, 15, 10, 47, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 10, 47, 0, time.UTC),
+			Usage:           time.Duration(4) * time.Second,
+			DisconnectCause: "200 OK",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "1877:893549741",
+			},
+			Cost: -1,
+		},
+		&engine.StoredCdr{
+			CgrId:           "f2f8d9341adfbbe1836b22f75182142061ef3d20",
+			TOR:             utils.VOICE,
+			AccId:           "3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:036e39a542d996f9",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 10, 57, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 10, 57, 0, time.UTC),
+			Usage:           time.Duration(4) * time.Second,
+			DisconnectCause: "200 OK",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "2407:1884881533",
+			},
+			Cost: -1,
+		},
+		&engine.StoredCdr{
+			CgrId:           "ccf05e7e3b9db9d2370bcbe316817447dba7df54",
+			TOR:             utils.VOICE,
+			AccId:           "a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:03111f3c949ca4c42",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 11, 30, 0, time.UTC), //2015-07-09T17:11:30+02:00
+			AnswerTime:      time.Date(2015, 7, 9, 15, 11, 30, 0, time.UTC),
+			Usage:           time.Duration(2) * time.Second,
+			DisconnectCause: "200 OK",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "3099:1909036290",
+			},
+			Cost: -1,
+		},
 	}
-	cdrRow := []string{"firstField", "secondField"}
-	_, err := cdrc.recordAsStoredCdr(cdrRow)
-	if err == nil {
-		t.Error("Failed to corectly detect missing fields from record")
-	}
-	cdrRow = []string{"acc1", "prepaid", "*out", "cgrates.org", "call", "1001", "1001", "+4986517174963", "2013-02-03 19:50:00", "2013-02-03 19:54:00", "62",
-		"supplier1", "172.16.1.1"}
-	rtCdr, err := cdrc.recordAsStoredCdr(cdrRow)
-	if err != nil {
-		t.Error("Failed to parse CDR in rated cdr", err)
-	}
-	expectedCdr := &utils.StoredCdr{
-		CgrId:       utils.FSCgrId(cdrRow[0]),
-		AccId:       cdrRow[0],
-		CdrSource:   cgrConfig.CdrcSourceId,
-		ReqType:     cdrRow[1],
-		Direction:   cdrRow[2],
-		Tenant:      cdrRow[3],
-		TOR:         cdrRow[4],
-		Account:     cdrRow[5],
-		Subject:     cdrRow[6],
-		Destination: cdrRow[7],
-		SetupTime:   time.Date(2013, 2, 3, 19, 50, 0, 0, time.UTC),
-		AnswerTime:  time.Date(2013, 2, 3, 19, 54, 0, 0, time.UTC),
-		Duration:    time.Duration(62) * time.Second,
-		ExtraFields: map[string]string{"supplier": "supplier1"},
-		Cost:        -1,
-	}
-	if !reflect.DeepEqual(expectedCdr, rtCdr) {
-		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
-	}
-	/*
-		if cdrAsForm.Get(utils.CDRSOURCE) != cgrConfig.CdrcSourceId {
-			t.Error("Unexpected cdrsource received", cdrAsForm.Get(utils.CDRSOURCE))
+
+	cdrFields := [][]*config.CfgCdrField{[]*config.CfgCdrField{
+		&config.CfgCdrField{Tag: "Tor", Type: utils.CDRFIELD, CdrFieldId: utils.TOR, Value: utils.ParseRSRFieldsMustCompile("^*voice", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "AccId", Type: utils.CDRFIELD, CdrFieldId: utils.ACCID, Mandatory: true},
+		&config.CfgCdrField{Tag: "ReqType", Type: utils.CDRFIELD, CdrFieldId: utils.REQTYPE, Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Tenant", Type: utils.CDRFIELD, CdrFieldId: utils.TENANT, Value: utils.ParseRSRFieldsMustCompile("^cgrates.org", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Category", Type: utils.CDRFIELD, CdrFieldId: utils.CATEGORY, Value: utils.ParseRSRFieldsMustCompile("^call", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Account", Type: utils.CDRFIELD, CdrFieldId: utils.ACCOUNT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Subject", Type: utils.CDRFIELD, CdrFieldId: utils.SUBJECT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Destination", Type: utils.CDRFIELD, CdrFieldId: utils.DESTINATION, Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "SetupTime", Type: utils.CDRFIELD, CdrFieldId: utils.SETUP_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "AnswerTime", Type: utils.CDRFIELD, CdrFieldId: utils.ANSWER_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Duration", Type: utils.CDRFIELD, CdrFieldId: utils.USAGE, Mandatory: true},
+		&config.CfgCdrField{Tag: "DisconnectCause", Type: utils.CDRFIELD, CdrFieldId: utils.DISCONNECT_CAUSE, Value: utils.ParseRSRFieldsMustCompile("4;^ ;5", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "DialogId", Type: utils.CDRFIELD, CdrFieldId: "DialogIdentifier", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP)},
+	}}
+	cdrc := &Cdrc{CdrFormat: utils.OSIPS_FLATSTORE, cdrSourceIds: []string{"TEST_CDRC"}, failedCallsPrefix: "missed_calls",
+		cdrFields: cdrFields, partialRecords: make(map[string]map[string]*PartialFlatstoreRecord),
+		guard: engine.NewGuardianLock()}
+	cdrsContent := bytes.NewReader([]byte(flatstoreCdrs))
+	csvReader := csv.NewReader(cdrsContent)
+	csvReader.Comma = '|'
+	cdrs := make([]*engine.StoredCdr, 0)
+	recNrs := 0
+	for {
+		recNrs++
+		cdrCsv, err := csvReader.Read()
+		if err != nil && err == io.EOF {
+			break // End of file
+		} else if err != nil {
+			t.Error("Unexpected error:", err)
 		}
-		if cdrAsForm.Get(utils.REQTYPE) != "prepaid" {
-			t.Error("Unexpected CDR value received", cdrAsForm.Get(utils.REQTYPE))
+		record, err := cdrc.processPartialRecord(cdrCsv, "dummyfilename")
+		if err != nil {
+			t.Error(err)
 		}
-		if cdrAsForm.Get("supplier") != "supplier1" {
-			t.Error("Unexpected CDR value received", cdrAsForm.Get("supplier"))
+		if record == nil {
+			continue // Partial record
 		}
-	*/
+		if storedCdr, err := cdrc.recordToStoredCdr(record, 0); err != nil {
+			t.Error(err)
+		} else if storedCdr != nil {
+			cdrs = append(cdrs, storedCdr)
+		}
+	}
+	if !reflect.DeepEqual(eCdrs, cdrs) {
+		t.Errorf("Expecting: %+v, received: %+v", eCdrs, cdrs)
+	}
+
 }
+
+func TestOsipsFlatstoreMissedCdrs(t *testing.T) {
+	flatstoreCdrs := `
+INVITE|ef6c6256|da501581|0bfdd176d1b93e7df3de5c6f4873ee04@0:0:0:0:0:0:0:0|487|Request Terminated|1436454643|*prepaid|1001|1002||1224:339382783
+INVITE|7905e511||81880da80a94bda81b425b09009e055c@0:0:0:0:0:0:0:0|404|Not Found|1436454668|*prepaid|1001|1002||1980:1216490844
+INVITE|324cb497|d4af7023|8deaadf2ae9a17809a391f05af31afb0@0:0:0:0:0:0:0:0|486|Busy here|1436454687|*postpaid|1002|1001||474:130115066
+`
+	eCdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{
+			CgrId:           "1c20aa6543a5a30d26b2354ae79e1f5fb720e8e5",
+			TOR:             utils.VOICE,
+			AccId:           "0bfdd176d1b93e7df3de5c6f4873ee04@0:0:0:0:0:0:0:0ef6c6256da501581",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 10, 43, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 10, 43, 0, time.UTC),
+			Usage:           0,
+			DisconnectCause: "487 Request Terminated",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "1224:339382783",
+			},
+			Cost: -1,
+		},
+		&engine.StoredCdr{
+			CgrId:           "054ab7c6c7fe6dc4a72f34e270027fa2aa930a58",
+			TOR:             utils.VOICE,
+			AccId:           "81880da80a94bda81b425b09009e055c@0:0:0:0:0:0:0:07905e511",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 11, 8, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 11, 8, 0, time.UTC),
+			Usage:           0,
+			DisconnectCause: "404 Not Found",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "1980:1216490844",
+			},
+			Cost: -1,
+		},
+		&engine.StoredCdr{
+			CgrId:           "d49ea63d1655b15149336004629f1cadd1434b89",
+			TOR:             utils.VOICE,
+			AccId:           "8deaadf2ae9a17809a391f05af31afb0@0:0:0:0:0:0:0:0324cb497d4af7023",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_POSTPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1002",
+			Subject:         "1002",
+			Destination:     "1001",
+			SetupTime:       time.Date(2015, 7, 9, 15, 11, 27, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 11, 27, 0, time.UTC),
+			Usage:           0,
+			DisconnectCause: "486 Busy here",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "474:130115066",
+			},
+			Cost: -1,
+		},
+	}
+
+	cdrFields := [][]*config.CfgCdrField{[]*config.CfgCdrField{
+		&config.CfgCdrField{Tag: "Tor", Type: utils.CDRFIELD, CdrFieldId: utils.TOR, Value: utils.ParseRSRFieldsMustCompile("^*voice", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "AccId", Type: utils.CDRFIELD, CdrFieldId: utils.ACCID, Mandatory: true},
+		&config.CfgCdrField{Tag: "ReqType", Type: utils.CDRFIELD, CdrFieldId: utils.REQTYPE, Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Tenant", Type: utils.CDRFIELD, CdrFieldId: utils.TENANT, Value: utils.ParseRSRFieldsMustCompile("^cgrates.org", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Category", Type: utils.CDRFIELD, CdrFieldId: utils.CATEGORY, Value: utils.ParseRSRFieldsMustCompile("^call", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Account", Type: utils.CDRFIELD, CdrFieldId: utils.ACCOUNT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Subject", Type: utils.CDRFIELD, CdrFieldId: utils.SUBJECT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Destination", Type: utils.CDRFIELD, CdrFieldId: utils.DESTINATION, Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "SetupTime", Type: utils.CDRFIELD, CdrFieldId: utils.SETUP_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "AnswerTime", Type: utils.CDRFIELD, CdrFieldId: utils.ANSWER_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Usage", Type: utils.CDRFIELD, CdrFieldId: utils.USAGE, Mandatory: true},
+		&config.CfgCdrField{Tag: "DisconnectCause", Type: utils.CDRFIELD, CdrFieldId: utils.DISCONNECT_CAUSE, Value: utils.ParseRSRFieldsMustCompile("4;^ ;5", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "DialogId", Type: utils.CDRFIELD, CdrFieldId: "DialogIdentifier", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP)},
+	}}
+	cdrc := &Cdrc{CdrFormat: utils.OSIPS_FLATSTORE, cdrSourceIds: []string{"TEST_CDRC"}, failedCallsPrefix: "missed_calls",
+		cdrFields: cdrFields, partialRecords: make(map[string]map[string]*PartialFlatstoreRecord),
+		guard: engine.NewGuardianLock()}
+	cdrsContent := bytes.NewReader([]byte(flatstoreCdrs))
+	csvReader := csv.NewReader(cdrsContent)
+	csvReader.Comma = '|'
+	cdrs := make([]*engine.StoredCdr, 0)
+	recNrs := 0
+	for {
+		recNrs++
+		cdrCsv, err := csvReader.Read()
+		if err != nil && err == io.EOF {
+			break // End of file
+		} else if err != nil {
+			t.Error("Unexpected error:", err)
+		}
+		record, err := cdrc.processPartialRecord(cdrCsv, "missed_calls_1.log")
+		if err != nil {
+			t.Error(err)
+		}
+		if record == nil {
+			continue // Partial record
+		}
+		if storedCdr, err := cdrc.recordToStoredCdr(record, 0); err != nil {
+			t.Error(err)
+		} else if storedCdr != nil {
+			cdrs = append(cdrs, storedCdr)
+		}
+	}
+	if !reflect.DeepEqual(eCdrs, cdrs) {
+		t.Errorf("Expecting: %+v, received: %+v", eCdrs, cdrs)
+	}
+
+}
+*/

cdrc/csv.go

@@ -0,0 +1,334 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func NewPartialFlatstoreRecord(record []string) (*PartialFlatstoreRecord, error) {
+	if len(record) < 7 {
+		return nil, errors.New("MISSING_IE")
+	}
+	pr := &PartialFlatstoreRecord{Method: record[0], AccId: record[3] + record[1] + record[2], Values: record}
+	var err error
+	if pr.Timestamp, err = utils.ParseTimeDetectLayout(record[6]); err != nil {
+		return nil, err
+	}
+	return pr, nil
+}
+
+// This is a partial record received from Flatstore, can be INVITE or BYE and it needs to be paired in order to produce duration
+type PartialFlatstoreRecord struct {
+	Method    string    // INVITE or BYE
+	AccId     string    // Copute here the AccId
+	Timestamp time.Time // Timestamp of the event, as written by db_flastore module
+	Values    []string  // Can contain original values or updated via UpdateValues
+}
+
+// Pairs INVITE and BYE into final record containing as last element the duration
+func pairToRecord(part1, part2 *PartialFlatstoreRecord) ([]string, error) {
+	var invite, bye *PartialFlatstoreRecord
+	if part1.Method == "INVITE" {
+		invite = part1
+	} else if part2.Method == "INVITE" {
+		invite = part2
+	} else {
+		return nil, errors.New("MISSING_INVITE")
+	}
+	if part1.Method == "BYE" {
+		bye = part1
+	} else if part2.Method == "BYE" {
+		bye = part2
+	} else {
+		return nil, errors.New("MISSING_BYE")
+	}
+	if len(invite.Values) != len(bye.Values) {
+		return nil, errors.New("INCONSISTENT_VALUES_LENGTH")
+	}
+	record := invite.Values
+	for idx := range record {
+		switch idx {
+		case 0, 1, 2, 3, 6: // Leave these values as they are
+		case 4, 5:
+			record[idx] = bye.Values[idx] // Update record with status from bye
+		default:
+			if bye.Values[idx] != "" { // Any value higher than 6 is dynamically inserted, overwrite if non empty
+				record[idx] = bye.Values[idx]
+			}
+
+		}
+	}
+	callDur := bye.Timestamp.Sub(invite.Timestamp)
+	record = append(record, strconv.FormatFloat(callDur.Seconds(), 'f', -1, 64))
+	return record, nil
+}
+
+func NewPartialRecordsCache(ttl time.Duration, cdrOutDir string, csvSep rune) (*PartialRecordsCache, error) {
+	return &PartialRecordsCache{ttl: ttl, cdrOutDir: cdrOutDir, csvSep: csvSep,
+		partialRecords: make(map[string]map[string]*PartialFlatstoreRecord), guard: engine.NewGuardianLock()}, nil
+}
+
+type PartialRecordsCache struct {
+	ttl            time.Duration
+	cdrOutDir      string
+	csvSep         rune
+	partialRecords map[string]map[string]*PartialFlatstoreRecord // [FileName"][AccId]*PartialRecord
+	guard          *engine.GuardianLock
+}
+
+// Dumps the cache into a .unpaired file in the outdir and cleans cache after
+func (self *PartialRecordsCache) dumpUnpairedRecords(fileName string) error {
+	_, err := self.guard.Guard(func() (interface{}, error) {
+		if len(self.partialRecords[fileName]) != 0 { // Only write the file if there are records in the cache
+			unpairedFilePath := path.Join(self.cdrOutDir, fileName+UNPAIRED_SUFFIX)
+			fileOut, err := os.Create(unpairedFilePath)
+			if err != nil {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed creating %s, error: %s", unpairedFilePath, err.Error()))
+				return nil, err
+			}
+			csvWriter := csv.NewWriter(fileOut)
+			csvWriter.Comma = self.csvSep
+			for _, pr := range self.partialRecords[fileName] {
+				if err := csvWriter.Write(pr.Values); err != nil {
+					engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed writing unpaired record %v to file: %s, error: %s", pr, unpairedFilePath, err.Error()))
+					return nil, err
+				}
+			}
+			csvWriter.Flush()
+		}
+		delete(self.partialRecords, fileName)
+		return nil, nil
+	}, fileName)
+	return err
+}
+
+// Search in cache and return the partial record with accountind id defined, prefFilename is searched at beginning because of better match probability
+func (self *PartialRecordsCache) GetPartialRecord(accId, prefFileName string) (string, *PartialFlatstoreRecord) {
+	var cachedFilename string
+	var cachedPartial *PartialFlatstoreRecord
+	checkCachedFNames := []string{prefFileName} // Higher probability to match as firstFileName
+	for fName := range self.partialRecords {
+		if fName != prefFileName {
+			checkCachedFNames = append(checkCachedFNames, fName)
+		}
+	}
+	for _, fName := range checkCachedFNames { // Need to lock them individually
+		self.guard.Guard(func() (interface{}, error) {
+			var hasPartial bool
+			if cachedPartial, hasPartial = self.partialRecords[fName][accId]; hasPartial {
+				cachedFilename = fName
+			}
+			return nil, nil
+		}, fName)
+		if cachedPartial != nil {
+			break
+		}
+	}
+	return cachedFilename, cachedPartial
+}
+
+func (self *PartialRecordsCache) CachePartial(fileName string, pr *PartialFlatstoreRecord) {
+	self.guard.Guard(func() (interface{}, error) {
+		if fileMp, hasFile := self.partialRecords[fileName]; !hasFile {
+			self.partialRecords[fileName] = map[string]*PartialFlatstoreRecord{pr.AccId: pr}
+			if self.ttl != 0 { // Schedule expiry/dump of the just created entry in cache
+				go func() {
+					time.Sleep(self.ttl)
+					self.dumpUnpairedRecords(fileName)
+				}()
+			}
+		} else if _, hasAccId := fileMp[pr.AccId]; !hasAccId {
+			self.partialRecords[fileName][pr.AccId] = pr
+		}
+		return nil, nil
+	}, fileName)
+}
+
+func (self *PartialRecordsCache) UncachePartial(fileName string, pr *PartialFlatstoreRecord) {
+	self.guard.Guard(func() (interface{}, error) {
+		delete(self.partialRecords[fileName], pr.AccId) // Remove the record out of cache
+		return nil, nil
+	}, fileName)
+}
+
+func NewCsvRecordsProcessor(csvReader *csv.Reader, cdrFormat, fileName, failedCallsPrefix string,
+	cdrSourceIds []string, duMultiplyFactors []float64, cdrFilters []utils.RSRFields, cdrFields [][]*config.CfgCdrField,
+	httpSkipTlsCheck bool, partialRecordsCache *PartialRecordsCache) *CsvRecordsProcessor {
+	return &CsvRecordsProcessor{csvReader: csvReader, cdrFormat: cdrFormat, fileName: fileName,
+		failedCallsPrefix: failedCallsPrefix, cdrSourceIds: cdrSourceIds,
+		duMultiplyFactors: duMultiplyFactors, cdrFilters: cdrFilters, cdrFields: cdrFields,
+		httpSkipTlsCheck: httpSkipTlsCheck, partialRecordsCache: partialRecordsCache}
+
+}
+
+type CsvRecordsProcessor struct {
+	csvReader           *csv.Reader
+	cdrFormat           string
+	fileName            string
+	failedCallsPrefix   string
+	cdrSourceIds        []string // Should be in sync with cdrFields on indexes
+	duMultiplyFactors   []float64
+	cdrFilters          []utils.RSRFields       // Should be in sync with cdrFields on indexes
+	cdrFields           [][]*config.CfgCdrField // Profiles directly connected with cdrFilters
+	httpSkipTlsCheck    bool
+	partialRecordsCache *PartialRecordsCache // Shared by cdrc so we can cache for all files in a folder
+}
+
+func (self *CsvRecordsProcessor) ProcessNextRecord() ([]*engine.StoredCdr, error) {
+	record, err := self.csvReader.Read()
+	if err != nil {
+		return nil, err
+	}
+	if utils.IsSliceMember([]string{utils.KAM_FLATSTORE, utils.OSIPS_FLATSTORE}, self.cdrFormat) {
+		if record, err = self.processPartialRecord(record); err != nil {
+			return nil, err
+		} else if record == nil {
+			return nil, nil // Due to partial, none returned
+		}
+	}
+	// Record was overwriten with complete information out of cache
+	return self.processRecord(record)
+}
+
+// Processes a single partial record for flatstore CDRs
+func (self *CsvRecordsProcessor) processPartialRecord(record []string) ([]string, error) {
+	if strings.HasPrefix(self.fileName, self.failedCallsPrefix) { // Use the first index since they should be the same in all configs
+		record = append(record, "0") // Append duration 0 for failed calls flatstore CDR and do not process it further
+		return record, nil
+	}
+	pr, err := NewPartialFlatstoreRecord(record)
+	if err != nil {
+		return nil, err
+	}
+	// Retrieve and complete the record from cache
+	cachedFilename, cachedPartial := self.partialRecordsCache.GetPartialRecord(pr.AccId, self.fileName)
+	if cachedPartial == nil { // Not cached, do it here and stop processing
+		self.partialRecordsCache.CachePartial(self.fileName, pr)
+		return nil, nil
+	}
+	pairedRecord, err := pairToRecord(cachedPartial, pr)
+	if err != nil {
+		return nil, err
+	}
+	self.partialRecordsCache.UncachePartial(cachedFilename, pr)
+	return pairedRecord, nil
+}
+
+// Takes the record from a slice and turns it into StoredCdrs, posting them to the cdrServer
+func (self *CsvRecordsProcessor) processRecord(record []string) ([]*engine.StoredCdr, error) {
+	recordCdrs := make([]*engine.StoredCdr, 0) // More CDRs based on the number of filters and field templates
+	for idx := range self.cdrFields {          // cdrFields coming from more templates will produce individual storCdr records
+		// Make sure filters are matching
+		filterBreak := false
+		for _, rsrFilter := range self.cdrFilters[idx] {
+			if rsrFilter == nil { // Nil filter does not need to match anything
+				continue
+			}
+			if cfgFieldIdx, _ := strconv.Atoi(rsrFilter.Id); len(record) <= cfgFieldIdx {
+				return nil, fmt.Errorf("Ignoring record: %v - cannot compile filter %+v", record, rsrFilter)
+			} else if !rsrFilter.FilterPasses(record[cfgFieldIdx]) {
+				filterBreak = true
+				break
+			}
+		}
+		if filterBreak { // Stop importing cdrc fields profile due to non matching filter
+			continue
+		}
+		if storedCdr, err := self.recordToStoredCdr(record, idx); err != nil {
+			return nil, fmt.Errorf("Failed converting to StoredCdr, error: %s", err.Error())
+		} else {
+			recordCdrs = append(recordCdrs, storedCdr)
+		}
+	}
+	return recordCdrs, nil
+}
+
+// Takes the record out of csv and turns it into storedCdr which can be processed by CDRS
+func (self *CsvRecordsProcessor) recordToStoredCdr(record []string, cfgIdx int) (*engine.StoredCdr, error) {
+	storedCdr := &engine.StoredCdr{CdrHost: "0.0.0.0", CdrSource: self.cdrSourceIds[cfgIdx], ExtraFields: make(map[string]string), Cost: -1}
+	var err error
+	var lazyHttpFields []*config.CfgCdrField
+	for _, cdrFldCfg := range self.cdrFields[cfgIdx] {
+		if utils.IsSliceMember([]string{utils.KAM_FLATSTORE, utils.OSIPS_FLATSTORE}, self.cdrFormat) { // Hardcode some values in case of flatstore
+			switch cdrFldCfg.CdrFieldId {
+			case utils.ACCID:
+				cdrFldCfg.Value = utils.ParseRSRFieldsMustCompile("3;1;2", utils.INFIELD_SEP) // in case of flatstore, accounting id is made up out of callid, from_tag and to_tag
+			case utils.USAGE:
+				cdrFldCfg.Value = utils.ParseRSRFieldsMustCompile(strconv.Itoa(len(record)-1), utils.INFIELD_SEP) // in case of flatstore, last element will be the duration computed by us
+			}
+
+		}
+		var fieldVal string
+		if cdrFldCfg.Type == utils.CDRFIELD {
+			for _, cfgFieldRSR := range cdrFldCfg.Value {
+				if cfgFieldRSR.IsStatic() {
+					fieldVal += cfgFieldRSR.ParseValue("")
+				} else { // Dynamic value extracted using index
+					if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx {
+						return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cdrFldCfg.Tag)
+					} else {
+						fieldVal += cfgFieldRSR.ParseValue(record[cfgFieldIdx])
+					}
+				}
+			}
+		} else if cdrFldCfg.Type == utils.HTTP_POST {
+			lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of storedCdr to http server
+		} else {
+			return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type)
+		}
+		if err := populateStoredCdrField(storedCdr, cdrFldCfg.CdrFieldId, fieldVal); err != nil {
+			return nil, err
+		}
+	}
+	storedCdr.CgrId = utils.Sha1(storedCdr.AccId, storedCdr.SetupTime.String())
+	if storedCdr.TOR == utils.DATA && self.duMultiplyFactors[cfgIdx] != 0 {
+		storedCdr.Usage = time.Duration(float64(storedCdr.Usage.Nanoseconds()) * self.duMultiplyFactors[cfgIdx])
+	}
+	for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields
+		var outValByte []byte
+		var fieldVal, httpAddr string
+		for _, rsrFld := range httpFieldCfg.Value {
+			httpAddr += rsrFld.ParseValue("")
+		}
+		if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, storedCdr); err != nil && httpFieldCfg.Mandatory {
+			return nil, err
+		} else {
+			fieldVal = string(outValByte)
+			if len(fieldVal) == 0 && httpFieldCfg.Mandatory {
+				return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag)
+			}
+			if err := populateStoredCdrField(storedCdr, httpFieldCfg.CdrFieldId, fieldVal); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return storedCdr, nil
+}

cdrc/csv_test.go

@@ -0,0 +1,151 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func TestCsvRecordForkCdr(t *testing.T) {
+	cgrConfig, _ := config.NewDefaultCGRConfig()
+	cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][utils.META_DEFAULT]
+	cdrcConfig.ContentFields = append(cdrcConfig.ContentFields, &config.CfgCdrField{Tag: "SupplierTest", Type: utils.CDRFIELD, CdrFieldId: "supplier", Value: []*utils.RSRField{&utils.RSRField{Id: "14"}}})
+	cdrcConfig.ContentFields = append(cdrcConfig.ContentFields, &config.CfgCdrField{Tag: "DisconnectCauseTest", Type: utils.CDRFIELD, CdrFieldId: utils.DISCONNECT_CAUSE,
+		Value: []*utils.RSRField{&utils.RSRField{Id: "16"}}})
+	csvProcessor := &CsvRecordsProcessor{cdrFormat: CSV, cdrSourceIds: []string{"TEST_CDRC"}, cdrFields: [][]*config.CfgCdrField{cdrcConfig.ContentFields}}
+	cdrRow := []string{"firstField", "secondField"}
+	_, err := csvProcessor.recordToStoredCdr(cdrRow, 0)
+	if err == nil {
+		t.Error("Failed to corectly detect missing fields from record")
+	}
+	cdrRow = []string{"ignored", "ignored", utils.VOICE, "acc1", utils.META_PREPAID, "*out", "cgrates.org", "call", "1001", "1001", "+4986517174963",
+		"2013-02-03 19:50:00", "2013-02-03 19:54:00", "62", "supplier1", "172.16.1.1", "NORMAL_DISCONNECT"}
+	rtCdr, err := csvProcessor.recordToStoredCdr(cdrRow, 0)
+	if err != nil {
+		t.Error("Failed to parse CDR in rated cdr", err)
+	}
+	expectedCdr := &engine.StoredCdr{
+		CgrId:           utils.Sha1(cdrRow[3], time.Date(2013, 2, 3, 19, 50, 0, 0, time.UTC).String()),
+		TOR:             cdrRow[2],
+		AccId:           cdrRow[3],
+		CdrHost:         "0.0.0.0", // Got it over internal interface
+		CdrSource:       "TEST_CDRC",
+		ReqType:         cdrRow[4],
+		Direction:       cdrRow[5],
+		Tenant:          cdrRow[6],
+		Category:        cdrRow[7],
+		Account:         cdrRow[8],
+		Subject:         cdrRow[9],
+		Destination:     cdrRow[10],
+		SetupTime:       time.Date(2013, 2, 3, 19, 50, 0, 0, time.UTC),
+		AnswerTime:      time.Date(2013, 2, 3, 19, 54, 0, 0, time.UTC),
+		Usage:           time.Duration(62) * time.Second,
+		Supplier:        "supplier1",
+		DisconnectCause: "NORMAL_DISCONNECT",
+		ExtraFields:     map[string]string{},
+		Cost:            -1,
+	}
+	if !reflect.DeepEqual(expectedCdr, rtCdr) {
+		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
+	}
+}
+
+func TestCsvDataMultiplyFactor(t *testing.T) {
+	cdrFields := []*config.CfgCdrField{&config.CfgCdrField{Tag: "TORField", Type: utils.CDRFIELD, CdrFieldId: "tor", Value: []*utils.RSRField{&utils.RSRField{Id: "0"}}},
+		&config.CfgCdrField{Tag: "UsageField", Type: utils.CDRFIELD, CdrFieldId: "usage", Value: []*utils.RSRField{&utils.RSRField{Id: "1"}}}}
+	csvProcessor := &CsvRecordsProcessor{cdrFormat: CSV, cdrSourceIds: []string{"TEST_CDRC"}, duMultiplyFactors: []float64{0}, cdrFields: [][]*config.CfgCdrField{cdrFields}}
+	cdrRow := []string{"*data", "1"}
+	rtCdr, err := csvProcessor.recordToStoredCdr(cdrRow, 0)
+	if err != nil {
+		t.Error("Failed to parse CDR in rated cdr", err)
+	}
+	var sTime time.Time
+	expectedCdr := &engine.StoredCdr{
+		CgrId:       utils.Sha1("", sTime.String()),
+		TOR:         cdrRow[0],
+		CdrHost:     "0.0.0.0",
+		CdrSource:   "TEST_CDRC",
+		Usage:       time.Duration(1) * time.Second,
+		ExtraFields: map[string]string{},
+		Cost:        -1,
+	}
+	if !reflect.DeepEqual(expectedCdr, rtCdr) {
+		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
+	}
+	csvProcessor.duMultiplyFactors = []float64{1024}
+	expectedCdr = &engine.StoredCdr{
+		CgrId:       utils.Sha1("", sTime.String()),
+		TOR:         cdrRow[0],
+		CdrHost:     "0.0.0.0",
+		CdrSource:   "TEST_CDRC",
+		Usage:       time.Duration(1024) * time.Second,
+		ExtraFields: map[string]string{},
+		Cost:        -1,
+	}
+	if rtCdr, _ := csvProcessor.recordToStoredCdr(cdrRow, 0); !reflect.DeepEqual(expectedCdr, rtCdr) {
+		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
+	}
+	cdrRow = []string{"*voice", "1"}
+	expectedCdr = &engine.StoredCdr{
+		CgrId:       utils.Sha1("", sTime.String()),
+		TOR:         cdrRow[0],
+		CdrHost:     "0.0.0.0",
+		CdrSource:   "TEST_CDRC",
+		Usage:       time.Duration(1) * time.Second,
+		ExtraFields: map[string]string{},
+		Cost:        -1,
+	}
+	if rtCdr, _ := csvProcessor.recordToStoredCdr(cdrRow, 0); !reflect.DeepEqual(expectedCdr, rtCdr) {
+		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
+	}
+}
+
+func TestCsvPairToRecord(t *testing.T) {
+	eRecord := []string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454408", "*prepaid", "1001", "1002", "", "3401:2069362475", "2"}
+	invPr := &PartialFlatstoreRecord{Method: "INVITE", Timestamp: time.Date(2015, 7, 9, 15, 6, 48, 0, time.UTC),
+		Values: []string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454408", "*prepaid", "1001", "1002", "", "3401:2069362475"}}
+	byePr := &PartialFlatstoreRecord{Method: "BYE", Timestamp: time.Date(2015, 7, 9, 15, 6, 50, 0, time.UTC),
+		Values: []string{"BYE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454410", "", "", "", "", "3401:2069362475"}}
+	if rec, err := pairToRecord(invPr, byePr); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eRecord, rec) {
+		t.Errorf("Expected: %+v, received: %+v", eRecord, rec)
+	}
+	if rec, err := pairToRecord(byePr, invPr); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eRecord, rec) {
+		t.Errorf("Expected: %+v, received: %+v", eRecord, rec)
+	}
+	if _, err := pairToRecord(byePr, byePr); err == nil || err.Error() != "MISSING_INVITE" {
+		t.Error(err)
+	}
+	if _, err := pairToRecord(invPr, invPr); err == nil || err.Error() != "MISSING_BYE" {
+		t.Error(err)
+	}
+	byePr.Values = []string{"BYE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454410", "", "", "", "3401:2069362475"} // Took one value out
+	if _, err := pairToRecord(invPr, byePr); err == nil || err.Error() != "INCONSISTENT_VALUES_LENGTH" {
+		t.Error(err)
+	}
+}

cdrc/flatstore_local_test.go

@@ -0,0 +1,161 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"io/ioutil"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+)
+
+var flatstoreCfgPath string
+var flatstoreCfg *config.CGRConfig
+var flatstoreRpc *rpc.Client
+var flatstoreCdrcCfg *config.CdrcConfig
+
+var fullSuccessfull = `INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475
+BYE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454410|||||3401:2069362475
+INVITE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454647|*postpaid|1002|1001||1877:893549741
+BYE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454651|||||1877:893549741
+INVITE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454657|*prepaid|1001|1002||2407:1884881533
+BYE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454661|||||2407:1884881533
+INVITE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454690|*prepaid|1001|1002||3099:1909036290
+BYE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454692|||||3099:1909036290
+`
+
+var fullMissed = `INVITE|ef6c6256|da501581|0bfdd176d1b93e7df3de5c6f4873ee04@0:0:0:0:0:0:0:0|487|Request Terminated|1436454643|*prepaid|1001|1002||1224:339382783
+INVITE|7905e511||81880da80a94bda81b425b09009e055c@0:0:0:0:0:0:0:0|404|Not Found|1436454668|*prepaid|1001|1002||1980:1216490844
+INVITE|324cb497|d4af7023|8deaadf2ae9a17809a391f05af31afb0@0:0:0:0:0:0:0:0|486|Busy here|1436454687|*postpaid|1002|1001||474:130115066`
+
+var part1 = `BYE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4ccb@0:0:0:0:0:0:0:0|200|OK|1436454651|||||1877:893549742
+`
+
+var part2 = `INVITE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4ccb@0:0:0:0:0:0:0:0|200|OK|1436454647|*postpaid|1002|1003||1877:893549742
+INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475`
+
+func TestFlatstoreLclInitCfg(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	flatstoreCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcflatstore")
+	if flatstoreCfg, err = config.NewCGRConfigFromFolder(flatstoreCfgPath); err != nil {
+		t.Fatal("Got config error: ", err.Error())
+	}
+}
+
+// InitDb so we can rely on count
+func TestFlatstoreLclInitCdrDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitStorDb(flatstoreCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Creates cdr files and moves them into processing folder
+func TestFlatstoreLclCreateCdrFiles(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if flatstoreCfg == nil {
+		t.Fatal("Empty default cdrc configuration")
+	}
+	flatstoreCdrcCfg = flatstoreCfg.CdrcProfiles["/tmp/cgr_flatstore/cdrc/in"]["FLATSTORE"]
+	if err := os.RemoveAll(flatstoreCdrcCfg.CdrInDir); err != nil {
+		t.Fatal("Error removing folder: ", flatstoreCdrcCfg.CdrInDir, err)
+	}
+	if err := os.MkdirAll(flatstoreCdrcCfg.CdrInDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", flatstoreCdrcCfg.CdrInDir, err)
+	}
+	if err := os.RemoveAll(flatstoreCdrcCfg.CdrOutDir); err != nil {
+		t.Fatal("Error removing folder: ", flatstoreCdrcCfg.CdrOutDir, err)
+	}
+	if err := os.MkdirAll(flatstoreCdrcCfg.CdrOutDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", flatstoreCdrcCfg.CdrOutDir, err)
+	}
+}
+
+func TestFlatstoreLclStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if _, err := engine.StopStartEngine(flatstoreCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestFlatstoreLclRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	flatstoreRpc, err = jsonrpc.Dial("tcp", flatstoreCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+func TestFlatstoreLclProcessFiles(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := ioutil.WriteFile(path.Join("/tmp", "acc_1.log"), []byte(fullSuccessfull), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	if err := ioutil.WriteFile(path.Join("/tmp", "missed_calls_1.log"), []byte(fullMissed), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	if err := ioutil.WriteFile(path.Join("/tmp", "acc_2.log"), []byte(part1), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	if err := ioutil.WriteFile(path.Join("/tmp", "acc_3.log"), []byte(part2), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	//Rename(oldpath, newpath string)
+	for _, fileName := range []string{"acc_1.log", "missed_calls_1.log", "acc_2.log", "acc_3.log"} {
+		if err := os.Rename(path.Join("/tmp", fileName), path.Join(flatstoreCdrcCfg.CdrInDir, fileName)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	time.Sleep(time.Duration(2) * time.Second) // Give time for processing to happen and the .unparired file to be written
+	filesInDir, _ := ioutil.ReadDir(flatstoreCdrcCfg.CdrInDir)
+	if len(filesInDir) != 0 {
+		t.Errorf("Files in cdrcInDir: %+v", filesInDir)
+	}
+	filesOutDir, _ := ioutil.ReadDir(flatstoreCdrcCfg.CdrOutDir)
+	if len(filesOutDir) != 5 {
+		t.Errorf("In CdrcOutDir, expecting 5 files, got: %d", len(filesOutDir))
+	}
+	ePartContent := "INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475\n"
+	if partContent, err := ioutil.ReadFile(path.Join(flatstoreCdrcCfg.CdrOutDir, "acc_3.log.unpaired")); err != nil {
+		t.Error(err)
+	} else if ePartContent != string(partContent) {
+		t.Errorf("Expecting: %s, received: %s", ePartContent, string(partContent))
+	}
+}

cdrc/fwv.go

@@ -0,0 +1,246 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func fwvValue(cdrLine string, indexStart, width int, padding string) string {
+	rawVal := cdrLine[indexStart : indexStart+width]
+	switch padding {
+	case "left":
+		rawVal = strings.TrimLeft(rawVal, " ")
+	case "right":
+		rawVal = strings.TrimRight(rawVal, " ")
+	case "zeroleft":
+		rawVal = strings.TrimLeft(rawVal, "0 ")
+	case "zeroright":
+		rawVal = strings.TrimRight(rawVal, "0 ")
+	}
+	return rawVal
+}
+
+func NewFwvRecordsProcessor(file *os.File, cdrcCfgs map[string]*config.CdrcConfig, dfltCfg *config.CdrcConfig, httpClient *http.Client, httpSkipTlsCheck bool) *FwvRecordsProcessor {
+	return &FwvRecordsProcessor{file: file, cdrcCfgs: cdrcCfgs, dfltCfg: dfltCfg, httpSkipTlsCheck: httpSkipTlsCheck}
+}
+
+type FwvRecordsProcessor struct {
+	file             *os.File
+	cdrcCfgs         map[string]*config.CdrcConfig
+	dfltCfg          *config.CdrcConfig // General parameters
+	httpClient       *http.Client
+	httpSkipTlsCheck bool
+	lineLen          int64             // Length of the line in the file
+	offset           int64             // Index of the next byte to process
+	trailerOffset    int64             // Index where trailer starts, to be used as boundary when reading cdrs
+	headerCdr        *engine.StoredCdr // Cache here the general purpose stored CDR
+}
+
+// Sets the line length based on first line, sets offset back to initial after reading
+func (self *FwvRecordsProcessor) setLineLen() error {
+	rdr := bufio.NewReader(self.file)
+	readBytes, err := rdr.ReadBytes('\n')
+	if err != nil {
+		return err
+	}
+	self.lineLen = int64(len(readBytes))
+	if _, err := self.file.Seek(0, 0); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (self *FwvRecordsProcessor) ProcessNextRecord() ([]*engine.StoredCdr, error) {
+	defer func() { self.offset += self.lineLen }() // Schedule increasing the offset once we are out from processing the record
+	if self.offset == 0 {                          // First time, set the necessary offsets
+		if err := self.setLineLen(); err != nil {
+			engine.Logger.Err(fmt.Sprintf("<Cdrc> Row 0, error: cannot set lineLen: %s", err.Error()))
+			return nil, io.EOF
+		}
+		if len(self.dfltCfg.TrailerFields) != 0 {
+			if fi, err := self.file.Stat(); err != nil {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Row 0, error: cannot get file stats: %s", err.Error()))
+				return nil, err
+			} else {
+				self.trailerOffset = fi.Size() - self.lineLen
+			}
+		}
+		if len(self.dfltCfg.HeaderFields) != 0 { // ToDo: Process here the header fields
+			if err := self.processHeader(); err != nil {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Row 0, error reading header: %s", err.Error()))
+				return nil, io.EOF
+			}
+			return nil, nil
+		}
+	}
+	recordCdrs := make([]*engine.StoredCdr, 0) // More CDRs based on the number of filters and field templates
+	if self.trailerOffset != 0 && self.offset >= self.trailerOffset {
+		if err := self.processTrailer(); err != nil && err != io.EOF {
+			engine.Logger.Err(fmt.Sprintf("<Cdrc> Read trailer error: %s ", err.Error()))
+		}
+		return nil, io.EOF
+	}
+	buf := make([]byte, self.lineLen)
+	nRead, err := self.file.Read(buf)
+	if err != nil {
+		return nil, err
+	} else if nRead != len(buf) {
+		engine.Logger.Err(fmt.Sprintf("<Cdrc> Could not read complete line, have instead: %s", string(buf)))
+		return nil, io.EOF
+	}
+	record := string(buf)
+	for cfgKey := range self.cdrcCfgs {
+		if passes := self.recordPassesCfgFilter(record, cfgKey); !passes {
+			continue
+		}
+		if storedCdr, err := self.recordToStoredCdr(record, cfgKey); err != nil {
+			return nil, fmt.Errorf("Failed converting to StoredCdr, error: %s", err.Error())
+		} else {
+			recordCdrs = append(recordCdrs, storedCdr)
+		}
+	}
+	return recordCdrs, nil
+}
+
+func (self *FwvRecordsProcessor) recordPassesCfgFilter(record, configKey string) bool {
+	filterPasses := true
+	for _, rsrFilter := range self.cdrcCfgs[configKey].CdrFilter {
+		if rsrFilter == nil { // Nil filter does not need to match anything
+			continue
+		}
+		if cfgFieldIdx, _ := strconv.Atoi(rsrFilter.Id); len(record) <= cfgFieldIdx {
+			fmt.Errorf("Ignoring record: %v - cannot compile filter %+v", record, rsrFilter)
+			return false
+		} else if !rsrFilter.FilterPasses(record[cfgFieldIdx:]) {
+			fmt.Printf("Record content to test: %s\n", record[cfgFieldIdx:])
+			filterPasses = false
+			break
+		}
+	}
+	return filterPasses
+}
+
+// Converts a record (header or normal) to StoredCdr
+func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cfgKey string) (*engine.StoredCdr, error) {
+	var err error
+	var lazyHttpFields []*config.CfgCdrField
+	var cfgFields []*config.CfgCdrField
+	var duMultiplyFactor float64
+	var storedCdr *engine.StoredCdr
+	if self.headerCdr != nil { // Clone the header CDR so we can use it as base to future processing (inherit fields defined there)
+		storedCdr = self.headerCdr.Clone()
+	} else {
+		storedCdr = &engine.StoredCdr{CdrHost: "0.0.0.0", ExtraFields: make(map[string]string), Cost: -1}
+	}
+	if cfgKey == "*header" {
+		cfgFields = self.dfltCfg.HeaderFields
+		storedCdr.CdrSource = self.dfltCfg.CdrSourceId
+		duMultiplyFactor = self.dfltCfg.DataUsageMultiplyFactor
+	} else {
+		cfgFields = self.cdrcCfgs[cfgKey].ContentFields
+		storedCdr.CdrSource = self.cdrcCfgs[cfgKey].CdrSourceId
+		duMultiplyFactor = self.cdrcCfgs[cfgKey].DataUsageMultiplyFactor
+	}
+	for _, cdrFldCfg := range cfgFields {
+		var fieldVal string
+		switch cdrFldCfg.Type {
+		case utils.CDRFIELD:
+			for _, cfgFieldRSR := range cdrFldCfg.Value {
+				if cfgFieldRSR.IsStatic() {
+					fieldVal += cfgFieldRSR.ParseValue("")
+				} else { // Dynamic value extracted using index
+					if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx {
+						return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cdrFldCfg.Tag)
+					} else {
+						fieldVal += cfgFieldRSR.ParseValue(fwvValue(record, cfgFieldIdx, cdrFldCfg.Width, cdrFldCfg.Padding))
+					}
+				}
+			}
+		case utils.HTTP_POST:
+			lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of storedCdr to http server
+		default:
+			//return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type)
+			continue // Don't do anything for unsupported fields
+		}
+		if err := populateStoredCdrField(storedCdr, cdrFldCfg.CdrFieldId, fieldVal); err != nil {
+			return nil, err
+		}
+	}
+	if storedCdr.CgrId == "" && storedCdr.AccId != "" && cfgKey != "*header" {
+		storedCdr.CgrId = utils.Sha1(storedCdr.AccId, storedCdr.SetupTime.String())
+	}
+	if storedCdr.TOR == utils.DATA && duMultiplyFactor != 0 {
+		storedCdr.Usage = time.Duration(float64(storedCdr.Usage.Nanoseconds()) * duMultiplyFactor)
+	}
+	for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields
+		var outValByte []byte
+		var fieldVal, httpAddr string
+		for _, rsrFld := range httpFieldCfg.Value {
+			httpAddr += rsrFld.ParseValue("")
+		}
+		if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, storedCdr); err != nil && httpFieldCfg.Mandatory {
+			return nil, err
+		} else {
+			fieldVal = string(outValByte)
+			if len(fieldVal) == 0 && httpFieldCfg.Mandatory {
+				return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag)
+			}
+			if err := populateStoredCdrField(storedCdr, httpFieldCfg.CdrFieldId, fieldVal); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return storedCdr, nil
+}
+
+func (self *FwvRecordsProcessor) processHeader() error {
+	buf := make([]byte, self.lineLen)
+	if nRead, err := self.file.Read(buf); err != nil {
+		return err
+	} else if nRead != len(buf) {
+		return fmt.Errorf("In header, line len: %d, have read: %d", self.lineLen, nRead)
+	}
+	var err error
+	if self.headerCdr, err = self.recordToStoredCdr(string(buf), "*header"); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (self *FwvRecordsProcessor) processTrailer() error {
+	buf := make([]byte, self.lineLen)
+	if nRead, err := self.file.ReadAt(buf, self.trailerOffset); err != nil {
+		return err
+	} else if nRead != len(buf) {
+		return fmt.Errorf("In trailer, line len: %d, have read: %d", self.lineLen, nRead)
+	}
+	//engine.Logger.Debug(fmt.Sprintf("Have read trailer: <%q>", string(buf)))
+	return nil
+}

cdrc/fwv_local_test.go

@@ -0,0 +1,150 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"io/ioutil"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"os"
+	"path"
+	"testing"
+	"time"
+)
+
+var fwvCfgPath string
+var fwvCfg *config.CGRConfig
+var fwvRpc *rpc.Client
+var fwvCdrcCfg *config.CdrcConfig
+
+var FW_CDR_FILE1 = `HDR0001DDB     ABC                                     Some Connect A.B.                       DDB-Some-10022-20120711-309.CDR         00030920120711100255                                                                    
+CDR0000010  0 20120708181506000123451234         0040123123120                  004                                            000018009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009
+CDR0000020  0 20120708190945000123451234         0040123123120                  004                                            000016009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009
+CDR0000030  0 20120708191009000123451234         0040123123120                  004                                            000020009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009
+CDR0000040  0 20120708231043000123451234         0040123123120                  004                                            000011009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009
+CDR0000050  0 20120709122216000123451235         004212                         004                                            000217009980010001ISDN  ABC   10Buiten uw regio                         HMR 00000000190000000000
+CDR0000060  0 20120709130542000123451236         0012323453                     004                                            000019009980010001ISDN  ABC   35Sterdiensten                            AP  00000000190000000000
+CDR0000070  0 20120709140032000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000080  0 20120709140142000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000090  0 20120709150305000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000100  0 20120709150414000123451237         0040012323453100               001                                            000057009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000110  0 20120709150531000123451237         0040012323453100               001                                            000059009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000120  0 20120709150635000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000130  0 20120709151756000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000140  0 20120709154549000123451237         0040012323453100               001                                            000052009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000150  0 20120709154701000123451237         0040012323453100               001                                            000121009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000160  0 20120709154842000123451237         0040012323453100               001                                            000055009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000170  0 20120709154956000123451237         0040012323453100               001                                            000115009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000180  0 20120709155131000123451237         0040012323453100               001                                            000059009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000190  0 20120709155236000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000200  0 20120709160309000123451237         0040012323453100               001                                            000100009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000210  0 20120709160415000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000220  0 20120709161739000123451237         0040012323453100               001                                            000058009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000230  0 20120709170356000123123459         0040123234531                  004                                            000012002760010001ISDN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000240  0 20120709181036000123123450         0012323453                     004                                            000042009980010001ISDN  ABC   05Binnen uw regio                         AP  00000010190000000010
+CDR0000250  0 20120709191245000123123458         0040123232350                  004                                            000012002760000001PSTN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000260  0 20120709202324000123123459         0040123234531                  004                                            000011002760010001ISDN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000270  0 20120709211756000123451237         0040012323453100               001                                            000051009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000280  0 20120709211852000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000290  0 20120709212904000123123458         0040123232350                  004                                            000012002760000001PSTN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000300  0 20120709073707000123123459         0040123234531                  004                                            000012002760010001ISDN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000310  0 20120709085451000123451237         0040012323453100               001                                            000744009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000320  0 20120709091756000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000330  0 20120710070434000123123458         0040123232350                  004                                            000012002760000001PSTN  276   10Buiten uw regio                         TB  00000009190000000009
+TRL0001DDB     ABC                                     Some Connect A.B.                       DDB-Some-10022-20120711-309.CDR         0003090000003300000030550000000001000000000100Y                                         
+`
+
+func TestFwvLclInitCfg(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	fwvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcfwv")
+	if fwvCfg, err = config.NewCGRConfigFromFolder(fwvCfgPath); err != nil {
+		t.Fatal("Got config error: ", err.Error())
+	}
+}
+
+// Creates cdr files and moves them into processing folder
+func TestFwvLclCreateCdrFiles(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if fwvCfg == nil {
+		t.Fatal("Empty default cdrc configuration")
+	}
+	fwvCdrcCfg = fwvCfg.CdrcProfiles["/tmp/cgr_fwv/cdrc/in"]["FWV1"]
+	if err := os.RemoveAll(fwvCdrcCfg.CdrInDir); err != nil {
+		t.Fatal("Error removing folder: ", fwvCdrcCfg.CdrInDir, err)
+	}
+	if err := os.MkdirAll(fwvCdrcCfg.CdrInDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", fwvCdrcCfg.CdrInDir, err)
+	}
+	if err := os.RemoveAll(fwvCdrcCfg.CdrOutDir); err != nil {
+		t.Fatal("Error removing folder: ", fwvCdrcCfg.CdrOutDir, err)
+	}
+	if err := os.MkdirAll(fwvCdrcCfg.CdrOutDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", fwvCdrcCfg.CdrOutDir, err)
+	}
+}
+
+func TestFwvLclStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if _, err := engine.StopStartEngine(fwvCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestFwvLclRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	fwvRpc, err = jsonrpc.Dial("tcp", fwvCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+func TestFwvLclProcessFiles(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	fileName := "test1.fwv"
+	if err := ioutil.WriteFile(path.Join("/tmp", fileName), []byte(FW_CDR_FILE1), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	if err := os.Rename(path.Join("/tmp", fileName), path.Join(fwvCdrcCfg.CdrInDir, fileName)); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Duration(1) * time.Second)
+	filesInDir, _ := ioutil.ReadDir(fwvCdrcCfg.CdrInDir)
+	if len(filesInDir) != 0 {
+		t.Errorf("Files in cdrcInDir: %d", len(filesInDir))
+	}
+	filesOutDir, _ := ioutil.ReadDir(fwvCdrcCfg.CdrOutDir)
+	if len(filesOutDir) != 1 {
+		t.Errorf("In CdrcOutDir, expecting 1 files, got: %d", len(filesOutDir))
+	}
+}

cdrc/fwv_test.go

@@ -0,0 +1,55 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/utils"
+	"testing"
+)
+
+func TestFwvValue(t *testing.T) {
+	cdrLine := "CDR0000010  0 20120708181506000123451234         0040123123120                  004                                            000018009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009"
+	if val := fwvValue(cdrLine, 30, 19, "right"); val != "0123451234" {
+		t.Errorf("Received: <%s>", val)
+	}
+	if val := fwvValue(cdrLine, 14, 16, "right"); val != "2012070818150600" { // SetupTime
+		t.Errorf("Received: <%s>", val)
+	}
+	if val := fwvValue(cdrLine, 127, 8, "right"); val != "00001800" { // Usage
+		t.Errorf("Received: <%s>", val)
+	}
+	cdrLine = "HDR0001DDB     ABC                                     Some Connect A.B.                       DDB-Some-10022-20120711-309.CDR         00030920120711100255                                                "
+	if val := fwvValue(cdrLine, 135, 6, "zeroleft"); val != "309" {
+		t.Errorf("Received: <%s>", val)
+	}
+
+}
+
+func TestFwvRecordPassesCfgFilter(t *testing.T) {
+	//record, configKey string) bool {
+	cgrConfig, _ := config.NewDefaultCGRConfig()
+	cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][utils.META_DEFAULT] // We don't really care that is for .csv since all we want to test are the filters
+	cdrcConfig.CdrFilter = utils.ParseRSRFieldsMustCompile(`~52:s/^0(\d{9})/+49${1}/(+49123123120)`, utils.INFIELD_SEP)
+	fwvRp := &FwvRecordsProcessor{cdrcCfgs: cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"]}
+	cdrLine := "CDR0000010  0 20120708181506000123451234         0040123123120                  004                                            000018009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009"
+	if passesFilter := fwvRp.recordPassesCfgFilter(cdrLine, utils.META_DEFAULT); !passesFilter {
+		t.Error("Not passes filter")
+	}
+}

cdre/cdrexporter.go

@@ -1,6 +1,6 @@
 /*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -19,10 +19,511 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package cdre
 
 import (
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/utils"
 )
 
-type CdrWriter interface {
-	WriteCdr(cdr *utils.StoredCdr) string
-	Close()
+const (
+	COST_DETAILS         = "cost_details"
+	DATETIME             = "datetime"
+	META_EXPORTID        = "*export_id"
+	META_TIMENOW         = "*time_now"
+	META_FIRSTCDRATIME   = "*first_cdr_atime"
+	META_LASTCDRATIME    = "*last_cdr_atime"
+	META_NRCDRS          = "*cdrs_number"
+	META_DURCDRS         = "*cdrs_duration"
+	META_SMSUSAGE        = "*sms_usage"
+	META_GENERICUSAGE    = "*generic_usage"
+	META_DATAUSAGE       = "*data_usage"
+	META_COSTCDRS        = "*cdrs_cost"
+	META_MASKDESTINATION = "*mask_destination"
+	META_FORMATCOST      = "*format_cost"
+)
+
+var err error
+
+func NewCdrExporter(cdrs []*engine.StoredCdr, cdrDb engine.CdrStorage, exportTpl *config.CdreConfig, cdrFormat string, fieldSeparator rune, exportId string,
+	dataUsageMultiplyFactor, smsUsageMultiplyFactor, genericUsageMultiplyFactor, costMultiplyFactor float64, costShiftDigits, roundDecimals, cgrPrecision int, maskDestId string, maskLen int, httpSkipTlsCheck bool) (*CdrExporter, error) {
+	if len(cdrs) == 0 { // Nothing to export
+		return nil, nil
+	}
+	cdre := &CdrExporter{
+		cdrs:                    cdrs,
+		cdrDb:                   cdrDb,
+		exportTemplate:          exportTpl,
+		cdrFormat:               cdrFormat,
+		fieldSeparator:          fieldSeparator,
+		exportId:                exportId,
+		dataUsageMultiplyFactor: dataUsageMultiplyFactor,
+		smsUsageMultiplyFactor:  smsUsageMultiplyFactor,
+		costMultiplyFactor:      costMultiplyFactor,
+		costShiftDigits:         costShiftDigits,
+		roundDecimals:           roundDecimals,
+		cgrPrecision:            cgrPrecision,
+		maskDestId:              maskDestId,
+		httpSkipTlsCheck:        httpSkipTlsCheck,
+		maskLen:                 maskLen,
+		negativeExports:         make(map[string]string),
+	}
+	if err := cdre.processCdrs(); err != nil {
+		return nil, err
+	}
+	return cdre, nil
+}
+
+type CdrExporter struct {
+	cdrs           []*engine.StoredCdr
+	cdrDb          engine.CdrStorage // Used to extract cost_details if these are requested
+	exportTemplate *config.CdreConfig
+	cdrFormat      string // csv, fwv
+	fieldSeparator rune
+	exportId       string // Unique identifier or this export
+	dataUsageMultiplyFactor,
+	smsUsageMultiplyFactor, // Multiply the SMS usage (eg: some billing systems billing them as minutes)
+	genericUsageMultiplyFactor,
+	costMultiplyFactor float64
+	costShiftDigits, roundDecimals, cgrPrecision                    int
+	maskDestId                                                      string
+	maskLen                                                         int
+	httpSkipTlsCheck                                                bool
+	header, trailer                                                 []string   // Header and Trailer fields
+	content                                                         [][]string // Rows of cdr fields
+	firstCdrATime, lastCdrATime                                     time.Time
+	numberOfRecords                                                 int
+	totalDuration, totalDataUsage, totalSmsUsage, totalGenericUsage time.Duration
+
+	totalCost                       float64
+	firstExpOrderId, lastExpOrderId int64
+	positiveExports                 []string          // CGRIds of successfully exported CDRs
+	negativeExports                 map[string]string // CgrIds of failed exports
+}
+
+// Return Json marshaled callCost attached to
+// Keep it separately so we test only this part in local tests
+func (cdre *CdrExporter) getCdrCostDetails(cgrId, runId string) (string, error) {
+	cc, err := cdre.cdrDb.GetCallCostLog(cgrId, "", runId)
+	if err != nil {
+		return "", err
+	} else if cc == nil {
+		return "", nil
+	}
+	ccJson, _ := json.Marshal(cc)
+	return string(ccJson), nil
+}
+
+func (cdre *CdrExporter) getCombimedCdrFieldVal(processedCdr *engine.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) {
+	var combinedVal string // Will result as combination of the field values, filters must match
+	for _, filterRule := range cfgCdrFld.FieldFilter {
+		fltrPass, ftrPassValue := processedCdr.PassesFieldFilter(filterRule)
+		if !fltrPass {
+			return "", nil
+		}
+		for _, cdr := range cdre.cdrs {
+			if cdr.CgrId != processedCdr.CgrId {
+				continue // We only care about cdrs with same primary cdr behind
+			}
+			if cdr.FieldAsString(&utils.RSRField{Id: filterRule.Id}) == ftrPassValue { // First CDR with filte
+				for _, rsrRule := range cfgCdrFld.Value {
+					combinedVal += cdr.FieldAsString(rsrRule)
+				}
+			}
+		}
+	}
+	return combinedVal, nil
+}
+
+// Check if the destination should be masked in output
+func (cdre *CdrExporter) maskedDestination(destination string) bool {
+	if len(cdre.maskDestId) != 0 && engine.CachedDestHasPrefix(cdre.maskDestId, destination) {
+		return true
+	}
+	return false
+}
+
+func (cdre *CdrExporter) getDateTimeFieldVal(cdr *engine.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) {
+	if len(cfgCdrFld.Value) == 0 {
+		return "", nil
+	}
+	for _, fltrRl := range cfgCdrFld.FieldFilter {
+		if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass {
+			return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl)
+		}
+	}
+	layout := cfgCdrFld.Layout
+	if len(layout) == 0 {
+		layout = time.RFC3339
+	}
+	if dtFld, err := utils.ParseTimeDetectLayout(cdr.FieldAsString(cfgCdrFld.Value[0])); err != nil { // Only one rule makes sense here
+		return "", err
+	} else {
+		return dtFld.Format(layout), nil
+	}
+}
+
+// Extracts the value specified by cfgHdr out of cdr
+func (cdre *CdrExporter) cdrFieldValue(cdr *engine.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) {
+	for _, fltrRl := range cfgCdrFld.FieldFilter {
+		if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass {
+			return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl)
+		}
+	}
+	layout := cfgCdrFld.Layout
+	if len(layout) == 0 {
+		layout = time.RFC3339
+	}
+	var retVal string // Concatenate the resulting values
+	for _, rsrFld := range cfgCdrFld.Value {
+		var cdrVal string
+		switch rsrFld.Id {
+		case COST_DETAILS: // Special case when we need to further extract cost_details out of logDb
+			if cdr.ExtraFields[COST_DETAILS], err = cdre.getCdrCostDetails(cdr.CgrId, cdr.MediationRunId); err != nil {
+				return "", err
+			} else {
+				cdrVal = cdr.FieldAsString(rsrFld)
+			}
+		case utils.COST:
+			cdrVal = cdr.FormatCost(cdre.costShiftDigits, cdre.roundDecimals)
+		case utils.USAGE:
+			cdrVal = cdr.FormatUsage(layout)
+		case utils.SETUP_TIME:
+			cdrVal = cdr.SetupTime.Format(layout)
+		case utils.ANSWER_TIME: // Format time based on layout
+			cdrVal = cdr.AnswerTime.Format(layout)
+		case utils.DESTINATION:
+			cdrVal = cdr.FieldAsString(rsrFld)
+			if cdre.maskLen != -1 && cdre.maskedDestination(cdrVal) {
+				cdrVal = MaskDestination(cdrVal, cdre.maskLen)
+			}
+		default:
+			cdrVal = cdr.FieldAsString(rsrFld)
+		}
+		retVal += cdrVal
+	}
+	return retVal, nil
+}
+
+// Handle various meta functions used in header/trailer
+func (cdre *CdrExporter) metaHandler(tag, arg string) (string, error) {
+	switch tag {
+	case META_EXPORTID:
+		return cdre.exportId, nil
+	case META_TIMENOW:
+		return time.Now().Format(arg), nil
+	case META_FIRSTCDRATIME:
+		return cdre.firstCdrATime.Format(arg), nil
+	case META_LASTCDRATIME:
+		return cdre.lastCdrATime.Format(arg), nil
+	case META_NRCDRS:
+		return strconv.Itoa(cdre.numberOfRecords), nil
+	case META_DURCDRS:
+		emulatedCdr := &engine.StoredCdr{TOR: utils.VOICE, Usage: cdre.totalDuration}
+		return emulatedCdr.FormatUsage(arg), nil
+	case META_SMSUSAGE:
+		emulatedCdr := &engine.StoredCdr{TOR: utils.SMS, Usage: cdre.totalSmsUsage}
+		return emulatedCdr.FormatUsage(arg), nil
+	case META_GENERICUSAGE:
+		emulatedCdr := &engine.StoredCdr{TOR: utils.GENERIC, Usage: cdre.totalGenericUsage}
+		return emulatedCdr.FormatUsage(arg), nil
+	case META_DATAUSAGE:
+		emulatedCdr := &engine.StoredCdr{TOR: utils.DATA, Usage: cdre.totalDataUsage}
+		return emulatedCdr.FormatUsage(arg), nil
+	case META_COSTCDRS:
+		return strconv.FormatFloat(utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE), 'f', -1, 64), nil
+	case META_MASKDESTINATION:
+		if cdre.maskedDestination(arg) {
+			return "1", nil
+		}
+		return "0", nil
+	default:
+		return "", fmt.Errorf("Unsupported METATAG: %s", tag)
+	}
+}
+
+// Compose and cache the header
+func (cdre *CdrExporter) composeHeader() error {
+	for _, cfgFld := range cdre.exportTemplate.HeaderFields {
+		var outVal string
+		switch cfgFld.Type {
+		case utils.FILLER:
+			outVal = cfgFld.Value.Id()
+			cfgFld.Padding = "right"
+		case utils.CONSTANT:
+			outVal = cfgFld.Value.Id()
+		case utils.METATAG:
+			outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cfgFld.Layout)
+		default:
+			return fmt.Errorf("Unsupported field type: %s", cfgFld.Type)
+		}
+		if err != nil {
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR header, field %s, error: %s", cfgFld.Tag, err.Error()))
+			return err
+		}
+		fmtOut := outVal
+		if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR header, field %s, error: %s", cfgFld.Tag, err.Error()))
+			return err
+		}
+		cdre.header = append(cdre.header, fmtOut)
+	}
+	return nil
+}
+
+// Compose and cache the trailer
+func (cdre *CdrExporter) composeTrailer() error {
+	for _, cfgFld := range cdre.exportTemplate.TrailerFields {
+		var outVal string
+		switch cfgFld.Type {
+		case utils.FILLER:
+			outVal = cfgFld.Value.Id()
+			cfgFld.Padding = "right"
+		case utils.CONSTANT:
+			outVal = cfgFld.Value.Id()
+		case utils.METATAG:
+			outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cfgFld.Layout)
+		default:
+			return fmt.Errorf("Unsupported field type: %s", cfgFld.Type)
+		}
+		if err != nil {
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, field: %s, error: %s", cfgFld.Tag, err.Error()))
+			return err
+		}
+		fmtOut := outVal
+		if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, field: %s, error: %s", cfgFld.Tag, err.Error()))
+			return err
+		}
+		cdre.trailer = append(cdre.trailer, fmtOut)
+	}
+	return nil
+}
+
+// Write individual cdr into content buffer, build stats
+func (cdre *CdrExporter) processCdr(cdr *engine.StoredCdr) error {
+	if cdr == nil || len(cdr.CgrId) == 0 { // We do not export empty CDRs
+		return nil
+	}
+	// Cost multiply
+	if cdre.dataUsageMultiplyFactor != 0.0 && cdr.TOR == utils.DATA {
+		cdr.UsageMultiply(cdre.dataUsageMultiplyFactor, cdre.cgrPrecision)
+	} else if cdre.smsUsageMultiplyFactor != 0 && cdr.TOR == utils.SMS {
+		cdr.UsageMultiply(cdre.smsUsageMultiplyFactor, cdre.cgrPrecision)
+	} else if cdre.genericUsageMultiplyFactor != 0 && cdr.TOR == utils.GENERIC {
+		cdr.UsageMultiply(cdre.genericUsageMultiplyFactor, cdre.cgrPrecision)
+	}
+	if cdre.costMultiplyFactor != 0.0 {
+		cdr.CostMultiply(cdre.costMultiplyFactor, cdre.cgrPrecision)
+	}
+	var err error
+	cdrRow := make([]string, len(cdre.exportTemplate.ContentFields))
+	for idx, cfgFld := range cdre.exportTemplate.ContentFields {
+		var outVal string
+		switch cfgFld.Type {
+		case utils.FILLER:
+			outVal = cfgFld.Value.Id()
+			cfgFld.Padding = "right"
+		case utils.CONSTANT:
+			outVal = cfgFld.Value.Id()
+		case utils.CDRFIELD:
+			outVal, err = cdre.cdrFieldValue(cdr, cfgFld)
+		case DATETIME:
+			outVal, err = cdre.getDateTimeFieldVal(cdr, cfgFld)
+		case utils.HTTP_POST:
+			var outValByte []byte
+			httpAddr := cfgFld.Value.Id()
+			if len(httpAddr) == 0 {
+				err = fmt.Errorf("Empty http address for field %s type %s", cfgFld.Tag, cfgFld.Type)
+			} else if outValByte, err = utils.HttpJsonPost(httpAddr, cdre.httpSkipTlsCheck, cdr); err == nil {
+				outVal = string(outValByte)
+				if len(outVal) == 0 && cfgFld.Mandatory {
+					err = fmt.Errorf("Empty result for http_post field: %s", cfgFld.Tag)
+				}
+			}
+		case utils.COMBIMED:
+			outVal, err = cdre.getCombimedCdrFieldVal(cdr, cfgFld)
+		case utils.METATAG:
+			if cfgFld.Value.Id() == META_MASKDESTINATION {
+				outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION}))
+			} else {
+				outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cfgFld.Layout)
+			}
+		}
+		if err != nil {
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s and runid: %s, error: %s", cdr.CgrId, cdr.MediationRunId, err.Error()))
+			return err
+		}
+		fmtOut := outVal
+		if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Tag, outVal, err.Error()))
+			return err
+		}
+		cdrRow[idx] += fmtOut
+	}
+	if len(cdrRow) == 0 { // No CDR data, most likely no configuration fields defined
+		return nil
+	} else {
+		cdre.content = append(cdre.content, cdrRow)
+	}
+	// Done with writing content, compute stats here
+	if cdre.firstCdrATime.IsZero() || cdr.AnswerTime.Before(cdre.firstCdrATime) {
+		cdre.firstCdrATime = cdr.AnswerTime
+	}
+	if cdr.AnswerTime.After(cdre.lastCdrATime) {
+		cdre.lastCdrATime = cdr.AnswerTime
+	}
+	cdre.numberOfRecords += 1
+	if cdr.TOR == utils.VOICE { // Only count duration for non data cdrs
+		cdre.totalDuration += cdr.Usage
+	}
+	if cdr.TOR == utils.SMS { // Count usage for SMS
+		cdre.totalSmsUsage += cdr.Usage
+	}
+	if cdr.TOR == utils.GENERIC { // Count usage for GENERIC
+		cdre.totalGenericUsage += cdr.Usage
+	}
+	if cdr.TOR == utils.DATA { // Count usage for DATA
+		cdre.totalDataUsage += cdr.Usage
+	}
+	if cdr.Cost != -1 {
+		cdre.totalCost += cdr.Cost
+		cdre.totalCost = utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE)
+	}
+	if cdre.firstExpOrderId > cdr.OrderId || cdre.firstExpOrderId == 0 {
+		cdre.firstExpOrderId = cdr.OrderId
+	}
+	if cdre.lastExpOrderId < cdr.OrderId {
+		cdre.lastExpOrderId = cdr.OrderId
+	}
+	return nil
+}
+
+// Builds header, content and trailers
+func (cdre *CdrExporter) processCdrs() error {
+	for _, cdr := range cdre.cdrs {
+		if err := cdre.processCdr(cdr); err != nil {
+			cdre.negativeExports[cdr.CgrId] = err.Error()
+		} else {
+			cdre.positiveExports = append(cdre.positiveExports, cdr.CgrId)
+		}
+	}
+	// Process header and trailer after processing cdrs since the metatag functions can access stats out of built cdrs
+	if cdre.exportTemplate.HeaderFields != nil {
+		if err := cdre.composeHeader(); err != nil {
+			return err
+		}
+	}
+	if cdre.exportTemplate.TrailerFields != nil {
+		if err := cdre.composeTrailer(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Simple write method
+func (cdre *CdrExporter) writeOut(ioWriter io.Writer) error {
+	if len(cdre.header) != 0 {
+		for _, fld := range append(cdre.header, "\n") {
+			if _, err := io.WriteString(ioWriter, fld); err != nil {
+				return err
+			}
+		}
+	}
+	for _, cdrContent := range cdre.content {
+		for _, cdrFld := range append(cdrContent, "\n") {
+			if _, err := io.WriteString(ioWriter, cdrFld); err != nil {
+				return err
+			}
+		}
+	}
+	if len(cdre.trailer) != 0 {
+		for _, fld := range append(cdre.trailer, "\n") {
+			if _, err := io.WriteString(ioWriter, fld); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// csvWriter specific method
+func (cdre *CdrExporter) writeCsv(csvWriter *csv.Writer) error {
+	csvWriter.Comma = cdre.fieldSeparator
+	if len(cdre.header) != 0 {
+		if err := csvWriter.Write(cdre.header); err != nil {
+			return err
+		}
+	}
+	for _, cdrContent := range cdre.content {
+		if err := csvWriter.Write(cdrContent); err != nil {
+			return err
+		}
+	}
+	if len(cdre.trailer) != 0 {
+		if err := csvWriter.Write(cdre.trailer); err != nil {
+			return err
+		}
+	}
+	csvWriter.Flush()
+	return nil
+}
+
+// General method to write the content out to a file
+func (cdre *CdrExporter) WriteToFile(filePath string) error {
+	fileOut, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	defer fileOut.Close()
+	switch cdre.cdrFormat {
+	case utils.DRYRUN:
+		return nil
+	case utils.CDRE_FIXED_WIDTH:
+		if err := cdre.writeOut(fileOut); err != nil {
+			return utils.NewErrServerError(err)
+		}
+	case utils.CSV:
+		csvWriter := csv.NewWriter(fileOut)
+		if err := cdre.writeCsv(csvWriter); err != nil {
+			return utils.NewErrServerError(err)
+		}
+	}
+	return nil
+}
+
+// Return the first exported Cdr OrderId
+func (cdre *CdrExporter) FirstOrderId() int64 {
+	return cdre.firstExpOrderId
+}
+
+// Return the last exported Cdr OrderId
+func (cdre *CdrExporter) LastOrderId() int64 {
+	return cdre.lastExpOrderId
+}
+
+// Return total cost in the exported cdrs
+func (cdre *CdrExporter) TotalCost() float64 {
+	return cdre.totalCost
+}
+
+func (cdre *CdrExporter) TotalExportedCdrs() int {
+	return cdre.numberOfRecords
+}
+
+// Return successfully exported CgrIds
+func (cdre *CdrExporter) PositiveExports() []string {
+	return cdre.positiveExports
+}
+
+// Return failed exported CgrIds together with the reason
+func (cdre *CdrExporter) NegativeExports() map[string]string {
+	return cdre.negativeExports
 }

cdre/cdrexporter_test.go

@@ -0,0 +1,124 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdre
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func TestCdreGetCombimedCdrFieldVal(t *testing.T) {
+	cfg, _ := config.NewDefaultCGRConfig()
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+			Usage: time.Duration(10) * time.Second, MediationRunId: "RUN_RTL", Cost: 1.01},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf2", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+			Usage: time.Duration(10) * time.Second, MediationRunId: "CUSTOMER1", Cost: 2.01},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+			Usage: time.Duration(10) * time.Second, MediationRunId: "CUSTOMER1", Cost: 3.01},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+			Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 4.01},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1000", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+			Usage: time.Duration(10) * time.Second, MediationRunId: "RETAIL1", Cost: 5.01},
+	}
+	cdre, err := NewCdrExporter(cdrs, nil, cfg.CdreProfiles["*default"], cfg.CdreProfiles["*default"].CdrFormat, cfg.CdreProfiles["*default"].FieldSeparator,
+		"firstexport", 0.0, 0.0, 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", 0, cfg.HttpSkipTlsVerify)
+	if err != nil {
+		t.Error("Unexpected error received: ", err)
+	}
+	fltrRule, _ := utils.ParseRSRFields("~mediation_runid:s/default/RUN_RTL/", utils.INFIELD_SEP)
+	val, _ := utils.ParseRSRFields("cost", utils.INFIELD_SEP)
+	cfgCdrFld := &config.CfgCdrField{Tag: "cost", Type: "cdrfield", CdrFieldId: "cost", Value: val, FieldFilter: fltrRule}
+	if costVal, err := cdre.getCombimedCdrFieldVal(cdrs[3], cfgCdrFld); err != nil {
+		t.Error(err)
+	} else if costVal != "1.01" {
+		t.Error("Expecting: 1.01, received: ", costVal)
+	}
+	fltrRule, _ = utils.ParseRSRFields("~mediation_runid:s/default/RETAIL1/", utils.INFIELD_SEP)
+	val, _ = utils.ParseRSRFields("account", utils.INFIELD_SEP)
+	cfgCdrFld = &config.CfgCdrField{Tag: "account", Type: "cdrfield", CdrFieldId: "account", Value: val, FieldFilter: fltrRule}
+	if acntVal, err := cdre.getCombimedCdrFieldVal(cdrs[3], cfgCdrFld); err != nil {
+		t.Error(err)
+	} else if acntVal != "1000" {
+		t.Error("Expecting: 1000, received: ", acntVal)
+	}
+}
+
+func TestGetDateTimeFieldVal(t *testing.T) {
+	cdreTst := new(CdrExporter)
+	cdrTst := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+		Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.01,
+		ExtraFields: map[string]string{"stop_time": "2014-06-11 19:19:00 +0000 UTC", "fieldextr2": "valextr2"}}
+	val, _ := utils.ParseRSRFields("stop_time", utils.INFIELD_SEP)
+	layout := "2006-01-02 15:04:05"
+	cfgCdrFld := &config.CfgCdrField{Tag: "stop_time", Type: "cdrfield", CdrFieldId: "stop_time", Value: val, Layout: layout}
+	if cdrVal, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err != nil {
+		t.Error(err)
+	} else if cdrVal != "2014-06-11 19:19:00" {
+		t.Error("Expecting: 2014-06-11 19:19:00, got: ", cdrVal)
+	}
+	// Test filter
+	fltr, _ := utils.ParseRSRFields("~tenant:s/(.+)/itsyscom.com/", utils.INFIELD_SEP)
+	cfgCdrFld = &config.CfgCdrField{Tag: "stop_time", Type: "cdrfield", CdrFieldId: "stop_time", Value: val, FieldFilter: fltr, Layout: layout}
+	if _, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err == nil {
+		t.Error(err)
+	}
+	val, _ = utils.ParseRSRFields("fieldextr2", utils.INFIELD_SEP)
+	cfgCdrFld = &config.CfgCdrField{Tag: "stop_time", Type: "cdrfield", CdrFieldId: "stop_time", Value: val, Layout: layout}
+	// Test time parse error
+	if _, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err == nil {
+		t.Error("Should give error here, got none.")
+	}
+}
+
+func TestCdreCdrFieldValue(t *testing.T) {
+	cdre := new(CdrExporter)
+	cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+		Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.01}
+	val, _ := utils.ParseRSRFields("destination", utils.INFIELD_SEP)
+	cfgCdrFld := &config.CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: val}
+	if val, err := cdre.cdrFieldValue(cdr, cfgCdrFld); err != nil {
+		t.Error(err)
+	} else if val != cdr.Destination {
+		t.Errorf("Expecting: %s, received: %s", cdr.Destination, val)
+	}
+	fltr, _ := utils.ParseRSRFields("~tenant:s/(.+)/itsyscom.com/", utils.INFIELD_SEP)
+	cfgCdrFld = &config.CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: val, FieldFilter: fltr}
+	if _, err := cdre.cdrFieldValue(cdr, cfgCdrFld); err == nil {
+		t.Error("Failed to use filter")
+	}
+}

cdre/csv.go

@@ -1,53 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package cdre
-
-import (
-	"encoding/csv"
-	"github.com/cgrates/cgrates/utils"
-	"io"
-)
-
-type CsvCdrWriter struct {
-	writer         *csv.Writer
-	roundDecimals  int               // Round floats like Cost using this number of decimals
-	exportedFields []*utils.RSRField // The fields exported, order important
-}
-
-func NewCsvCdrWriter(writer io.Writer, roundDecimals int, exportedFields []*utils.RSRField) *CsvCdrWriter {
-	return &CsvCdrWriter{csv.NewWriter(writer), roundDecimals, exportedFields}
-}
-
-func (csvwr *CsvCdrWriter) WriteCdr(cdr *utils.StoredCdr) error {
-	row := make([]string, len(csvwr.exportedFields))
-	for idx, fld := range csvwr.exportedFields {
-		var fldVal string
-		if fld.Id == utils.COST {
-			fldVal = cdr.FormatCost(csvwr.roundDecimals)
-		} else {
-			fldVal = cdr.ExportFieldValue(fld.Id)
-		}
-		row[idx] = fld.ParseValue(fldVal)
-	}
-	return csvwr.writer.Write(row)
-}
-
-func (csvwr *CsvCdrWriter) Close() {
-	csvwr.writer.Flush()
-}

cdre/csv_test.go

@@ -1,6 +1,6 @@
 /*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -20,28 +20,67 @@ package cdre
 
 import (
 	"bytes"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/utils"
+	"encoding/csv"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
 )
 
 func TestCsvCdrWriter(t *testing.T) {
 	writer := &bytes.Buffer{}
 	cfg, _ := config.NewDefaultCGRConfig()
-	exportedFields := append(cfg.CdreExportedFields, &utils.RSRField{Id: "extra3"}, &utils.RSRField{Id: "dummy_extra"}, &utils.RSRField{Id: "extra1"})
-	csvCdrWriter := NewCsvCdrWriter(writer, 4, exportedFields)
-	ratedCdr := &utils.StoredCdr{CgrId: utils.FSCgrId("dsafdsaf"), AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
-		TOR: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
-		Duration: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID,
+	storedCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+		Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID,
 		ExtraFields: map[string]string{"extra1": "val_extra1", "extra2": "val_extra2", "extra3": "val_extra3"}, Cost: 1.01,
 	}
-	csvCdrWriter.WriteCdr(ratedCdr)
-	csvCdrWriter.Close()
-	expected := `b18944ef4dc618569f24c27b9872827a242bad0c,default,dsafdsaf,192.168.1.1,rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07 08:42:25 +0000 UTC,2013-11-07 08:42:26 +0000 UTC,10,1.0100,val_extra3,"",val_extra1`
+	cdre, err := NewCdrExporter([]*engine.StoredCdr{storedCdr1}, nil, cfg.CdreProfiles["*default"], utils.CSV, ',', "firstexport", 0.0, 0.0, 0.0, 0.0, 0, 4,
+		cfg.RoundingDecimals, "", 0, cfg.HttpSkipTlsVerify)
+	if err != nil {
+		t.Error("Unexpected error received: ", err)
+	}
+	csvWriter := csv.NewWriter(writer)
+	if err := cdre.writeCsv(csvWriter); err != nil {
+		t.Error("Unexpected error: ", err)
+	}
+	expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6,*default,*voice,dsafdsaf,*rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07T08:42:25Z,2013-11-07T08:42:26Z,10,1.0100`
 	result := strings.TrimSpace(writer.String())
 	if result != expected {
 		t.Errorf("Expected: \n%s received: \n%s.", expected, result)
 	}
+	if cdre.TotalCost() != 1.01 {
+		t.Error("Unexpected TotalCost: ", cdre.TotalCost())
+	}
+}
+
+func TestAlternativeFieldSeparator(t *testing.T) {
+	writer := &bytes.Buffer{}
+	cfg, _ := config.NewDefaultCGRConfig()
+	storedCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+		Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID,
+		ExtraFields: map[string]string{"extra1": "val_extra1", "extra2": "val_extra2", "extra3": "val_extra3"}, Cost: 1.01,
+	}
+	cdre, err := NewCdrExporter([]*engine.StoredCdr{storedCdr1}, nil, cfg.CdreProfiles["*default"], utils.CSV, '|', "firstexport", 0.0, 0.0, 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", 0, cfg.HttpSkipTlsVerify)
+	if err != nil {
+		t.Error("Unexpected error received: ", err)
+	}
+	csvWriter := csv.NewWriter(writer)
+	if err := cdre.writeCsv(csvWriter); err != nil {
+		t.Error("Unexpected error: ", err)
+	}
+	expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6|*default|*voice|dsafdsaf|*rated|*out|cgrates.org|call|1001|1001|1002|2013-11-07T08:42:25Z|2013-11-07T08:42:26Z|10|1.0100`
+	result := strings.TrimSpace(writer.String())
+	if result != expected {
+		t.Errorf("Expected: \n%s received: \n%s.", expected, result)
+	}
+	if cdre.TotalCost() != 1.01 {
+		t.Error("Unexpected TotalCost: ", cdre.TotalCost())
+	}
 }

cdre/fixedwidth.go

@@ -1,267 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package cdre
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
-	"io"
-	"os"
-	"strconv"
-	"strings"
-	"time"
-)
-
-const (
-	COST_DETAILS          = "cost_details"
-	FILLER                = "filler"
-	CONSTANT              = "constant"
-	CDRFIELD              = "cdrfield"
-	METATAG               = "metatag"
-	CONCATENATED_CDRFIELD = "concatenated_cdrfield"
-	META_EXPORTID         = "export_id"
-	META_TIMENOW          = "time_now"
-	META_FIRSTCDRTIME     = "first_cdr_time"
-	META_LASTCDRTIME      = "last_cdr_time"
-	META_NRCDRS           = "cdrs_number"
-	META_DURCDRS          = "cdrs_duration"
-	META_COSTCDRS         = "cdrs_cost"
-)
-
-var err error
-
-func NewFWCdrWriter(logDb engine.LogStorage, outFile *os.File, exportTpl *config.CgrXmlCdreFwCfg, exportId string, roundDecimals int) (*FixedWidthCdrWriter, error) {
-	return &FixedWidthCdrWriter{
-		logDb:          logDb,
-		writer:         outFile,
-		exportTemplate: exportTpl,
-		exportId:       exportId,
-		roundDecimals:  roundDecimals,
-		header:         &bytes.Buffer{},
-		content:        &bytes.Buffer{},
-		trailer:        &bytes.Buffer{}}, nil
-}
-
-type FixedWidthCdrWriter struct {
-	logDb                     engine.LogStorage // Used to extract cost_details if these are requested
-	writer                    io.Writer
-	exportTemplate            *config.CgrXmlCdreFwCfg
-	exportId                  string // Unique identifier or this export
-	roundDecimals             int
-	header, content, trailer  *bytes.Buffer
-	firstCdrTime, lastCdrTime time.Time
-	numberOfRecords           int
-	totalDuration             time.Duration
-	totalCost                 float64
-}
-
-// Return Json marshaled callCost attached to
-// Keep it separately so we test only this part in local tests
-func (fww *FixedWidthCdrWriter) getCdrCostDetails(cgrId, runId string) (string, error) {
-	cc, err := fww.logDb.GetCallCostLog(cgrId, "", runId)
-	if err != nil {
-		return "", err
-	} else if cc == nil {
-		return "", nil
-	}
-	ccJson, _ := json.Marshal(cc)
-	return string(ccJson), nil
-}
-
-// Extracts the value specified by cfgHdr out of cdr
-func (fww *FixedWidthCdrWriter) cdrFieldValue(cdr *utils.StoredCdr, cfgHdr, layout string) (string, error) {
-	rsrField, err := utils.NewRSRField(cfgHdr)
-	if err != nil {
-		return "", err
-	} else if rsrField == nil {
-		return "", nil
-	}
-	var cdrVal string
-	switch rsrField.Id {
-	case COST_DETAILS: // Special case when we need to further extract cost_details out of logDb
-		if cdrVal, err = fww.getCdrCostDetails(cdr.CgrId, cdr.MediationRunId); err != nil {
-			return "", err
-		}
-	case utils.COST:
-		cdrVal = cdr.FormatCost(fww.roundDecimals)
-	case utils.SETUP_TIME:
-		cdrVal = cdr.SetupTime.Format(layout)
-	case utils.ANSWER_TIME: // Format time based on layout
-		cdrVal = cdr.AnswerTime.Format(layout)
-	default:
-		cdrVal = cdr.ExportFieldValue(rsrField.Id)
-	}
-	return rsrField.ParseValue(cdrVal), nil
-}
-
-func (fww *FixedWidthCdrWriter) metaHandler(tag, layout string) (string, error) {
-	switch tag {
-	case META_EXPORTID:
-		return fww.exportId, nil
-	case META_TIMENOW:
-		return time.Now().Format(layout), nil
-	case META_FIRSTCDRTIME:
-		return fww.firstCdrTime.Format(layout), nil
-	case META_LASTCDRTIME:
-		return fww.lastCdrTime.Format(layout), nil
-	case META_NRCDRS:
-		return strconv.Itoa(fww.numberOfRecords), nil
-	case META_DURCDRS:
-		return strconv.FormatFloat(fww.totalDuration.Seconds(), 'f', -1, 64), nil
-	case META_COSTCDRS:
-		return strconv.FormatFloat(utils.Round(fww.totalCost, fww.roundDecimals, utils.ROUNDING_MIDDLE), 'f', -1, 64), nil
-	default:
-		return "", errors.New("Unsupported METATAG")
-	}
-	return "", nil
-}
-
-// Writes the header into it's buffer
-func (fww *FixedWidthCdrWriter) ComposeHeader() error {
-	header := ""
-	for _, cfgFld := range fww.exportTemplate.Header.Fields {
-		var outVal string
-		switch cfgFld.Type {
-		case FILLER, CONSTANT:
-			outVal = cfgFld.Value
-		case METATAG:
-			outVal, err = fww.metaHandler(cfgFld.Value, cfgFld.Layout)
-		default:
-			return fmt.Errorf("Unsupported field type: %s", cfgFld.Type)
-		}
-		if err != nil {
-			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR header, error: %s", err.Error()))
-			return err
-		}
-		if fmtOut, err := FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
-			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR header, error: %s", err.Error()))
-			return err
-		} else {
-			header += fmtOut
-		}
-	}
-	if len(header) == 0 { // No header data, most likely no configuration fields defined
-		return nil
-	}
-	header += "\n" // Done with cdr, postpend new line char
-	fww.header.WriteString(header)
-	return nil
-}
-
-// Writes the trailer into it's buffer
-func (fww *FixedWidthCdrWriter) ComposeTrailer() error {
-	trailer := ""
-	for _, cfgFld := range fww.exportTemplate.Trailer.Fields {
-		var outVal string
-		switch cfgFld.Type {
-		case FILLER, CONSTANT:
-			outVal = cfgFld.Value
-		case METATAG:
-			outVal, err = fww.metaHandler(cfgFld.Value, cfgFld.Layout)
-		default:
-			return fmt.Errorf("Unsupported field type: %s", cfgFld.Type)
-		}
-		if err != nil {
-			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, error: %s", err.Error()))
-			return err
-		}
-		if fmtOut, err := FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
-			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, error: %s", err.Error()))
-			return err
-		} else {
-			trailer += fmtOut
-		}
-	}
-	if len(trailer) == 0 { // No header data, most likely no configuration fields defined
-		return nil
-	}
-	trailer += "\n" // Done with cdr, postpend new line char
-	fww.trailer.WriteString(trailer)
-	return nil
-}
-
-// Write individual cdr into content buffer, build stats
-func (fww *FixedWidthCdrWriter) WriteCdr(cdr *utils.StoredCdr) error {
-	if cdr == nil || len(cdr.CgrId) == 0 { // We do not export empty CDRs
-		return nil
-	}
-	var err error
-	cdrRow := ""
-	for _, cfgFld := range fww.exportTemplate.Content.Fields {
-		var outVal string
-		switch cfgFld.Type {
-		case FILLER, CONSTANT:
-			outVal = cfgFld.Value
-		case CDRFIELD:
-			outVal, err = fww.cdrFieldValue(cdr, cfgFld.Value, cfgFld.Layout)
-		case CONCATENATED_CDRFIELD:
-			for _, fld := range strings.Split(cfgFld.Value, ",") {
-				if fldOut, err := fww.cdrFieldValue(cdr, fld, cfgFld.Layout); err != nil {
-					break // The error will be reported bellow
-				} else {
-					outVal += fldOut
-				}
-			}
-		}
-		if err != nil {
-			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s and runid: %s, error: %s", cdr.CgrId, cdr.MediationRunId, err.Error()))
-			return err
-		}
-		if fmtOut, err := FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
-			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s and runid: %s, error: %s", cdr.CgrId, cdr.MediationRunId, err.Error()))
-			return err
-		} else {
-			cdrRow += fmtOut
-		}
-	}
-	if len(cdrRow) == 0 { // No CDR data, most likely no configuration fields defined
-		return nil
-	}
-	cdrRow += "\n" // Done with cdr, postpend new line char
-	fww.content.WriteString(cdrRow)
-	// Done with writing content, compute stats here
-	if fww.firstCdrTime.IsZero() || cdr.SetupTime.Before(fww.firstCdrTime) {
-		fww.firstCdrTime = cdr.SetupTime
-	}
-	if cdr.SetupTime.After(fww.lastCdrTime) {
-		fww.lastCdrTime = cdr.SetupTime
-	}
-	fww.numberOfRecords += 1
-	fww.totalDuration += cdr.Duration
-	fww.totalCost += cdr.Cost
-	fww.totalCost = utils.Round(fww.totalCost, fww.roundDecimals, utils.ROUNDING_MIDDLE)
-	return nil
-}
-
-func (fww *FixedWidthCdrWriter) Close() {
-	if fww.exportTemplate.Header != nil {
-		fww.ComposeHeader()
-	}
-	if fww.exportTemplate.Trailer != nil {
-		fww.ComposeTrailer()
-	}
-	for _, buf := range []*bytes.Buffer{fww.header, fww.content, fww.trailer} {
-		fww.writer.Write(buf.Bytes())
-	}
-}

cdre/fixedwidth_test.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -20,91 +20,123 @@ package cdre
 
 import (
 	"bytes"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/utils"
 	"math"
 	"testing"
 	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
 )
 
-var hdrCfgFlds = []*config.CgrXmlCfgCdrField{
-	&config.CgrXmlCfgCdrField{Name: "TypeOfRecord", Type: CONSTANT, Value: "10", Width: 2},
-	&config.CgrXmlCfgCdrField{Name: "Filler1", Type: FILLER, Width: 3, Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "DistributorCode", Type: CONSTANT, Value: "VOI", Width: 3},
-	&config.CgrXmlCfgCdrField{Name: "FileSeqNr", Type: METATAG, Value: "export_id", Width: 5, Strip: "right", Padding: "zeroleft"},
-	&config.CgrXmlCfgCdrField{Name: "LastCdr", Type: METATAG, Value: "last_cdr_time", Width: 12, Layout: "020106150400"},
-	&config.CgrXmlCfgCdrField{Name: "FileCreationfTime", Type: METATAG, Value: "time_now", Width: 12, Layout: "020106150400"},
-	&config.CgrXmlCfgCdrField{Name: "FileVersion", Type: CONSTANT, Value: "01", Width: 2},
-	&config.CgrXmlCfgCdrField{Name: "Filler2", Type: FILLER, Width: 105, Padding: "right"},
+var hdrJsnCfgFlds = []*config.CdrFieldJsonCfg{
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TypeOfRecord"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("10"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler1"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(3)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("DistributorCode"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("VOI"), Width: utils.IntPointer(3)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FileSeqNr"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_EXPORTID),
+		Width: utils.IntPointer(5), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("zeroleft")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("LastCdr"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_LASTCDRATIME),
+		Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FileCreationfTime"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_TIMENOW),
+		Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FileVersion"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("01"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler2"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(105)},
 }
 
-var contentCfgFlds = []*config.CgrXmlCfgCdrField{
-	&config.CgrXmlCfgCdrField{Name: "TypeOfRecord", Type: CONSTANT, Value: "20", Width: 2},
-	&config.CgrXmlCfgCdrField{Name: "Account", Type: CDRFIELD, Value: utils.ACCOUNT, Width: 12, Strip: "left", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "Subject", Type: CDRFIELD, Value: utils.SUBJECT, Width: 5, Strip: "right", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "CLI", Type: CDRFIELD, Value: "cli", Width: 15, Strip: "xright", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "Destination", Type: CDRFIELD, Value: utils.DESTINATION, Width: 24, Strip: "xright", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "TOR", Type: CONSTANT, Value: "02", Width: 2},
-	&config.CgrXmlCfgCdrField{Name: "SubtypeTOR", Type: CONSTANT, Value: "11", Width: 4, Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "SetupTime", Type: CDRFIELD, Value: utils.SETUP_TIME, Width: 12, Strip: "right", Padding: "right", Layout: "020106150400"},
-	&config.CgrXmlCfgCdrField{Name: "Duration", Type: CDRFIELD, Value: utils.DURATION, Width: 6, Strip: "right", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "DataVolume", Type: FILLER, Width: 6, Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "TaxCode", Type: CONSTANT, Value: "1", Width: 1},
-	&config.CgrXmlCfgCdrField{Name: "OperatorCode", Type: CDRFIELD, Value: "opercode", Width: 2, Strip: "right", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "ProductId", Type: CDRFIELD, Value: "productid", Width: 5, Strip: "right", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "NetworkId", Type: CONSTANT, Value: "3", Width: 1},
-	&config.CgrXmlCfgCdrField{Name: "CallId", Type: CDRFIELD, Value: utils.ACCID, Width: 16, Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "Filler", Type: FILLER, Width: 8, Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "Filler", Type: FILLER, Width: 8, Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "TerminationCode", Type: CONCATENATED_CDRFIELD, Value: "operator,product", Width: 5, Strip: "right", Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "Cost", Type: CDRFIELD, Value: utils.COST, Width: 9, Padding: "zeroleft"},
-	&config.CgrXmlCfgCdrField{Name: "DestinationPrivacy", Type: CDRFIELD, Value: "destination_privacy", Width: 1, Strip: "right", Padding: "right"},
+var contentJsnCfgFlds = []*config.CdrFieldJsonCfg{
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TypeOfRecord"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("20"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Account"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.ACCOUNT), Width: utils.IntPointer(12),
+		Strip: utils.StringPointer("left"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Subject"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.SUBJECT), Width: utils.IntPointer(5),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("CLI"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("cli"), Width: utils.IntPointer(15),
+		Strip: utils.StringPointer("xright"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Destination"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.DESTINATION), Width: utils.IntPointer(24),
+		Strip: utils.StringPointer("xright"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TOR"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("02"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("SubtypeTOR"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("11"), Width: utils.IntPointer(4),
+		Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("SetupTime"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.SETUP_TIME), Width: utils.IntPointer(12),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right"), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Duration"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.USAGE), Width: utils.IntPointer(6),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right"), Layout: utils.StringPointer(utils.SECONDS)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("DataVolume"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(6)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TaxCode"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("1"), Width: utils.IntPointer(1)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("OperatorCode"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("opercode"), Width: utils.IntPointer(2),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("ProductId"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("productid"), Width: utils.IntPointer(5),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("NetworkId"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("3"), Width: utils.IntPointer(1)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("CallId"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.ACCID), Width: utils.IntPointer(16),
+		Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(8)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(8)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TerminationCode"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("operator;product"),
+		Width: utils.IntPointer(5), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Cost"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.COST), Width: utils.IntPointer(9),
+		Padding: utils.StringPointer("zeroleft")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("DestinationPrivacy"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_MASKDESTINATION),
+		Width: utils.IntPointer(1)},
 }
 
-var trailerCfgFlds = []*config.CgrXmlCfgCdrField{
-	&config.CgrXmlCfgCdrField{Name: "TypeOfRecord", Type: CONSTANT, Value: "90", Width: 2},
-	&config.CgrXmlCfgCdrField{Name: "Filler1", Type: FILLER, Width: 3, Padding: "right"},
-	&config.CgrXmlCfgCdrField{Name: "DistributorCode", Type: CONSTANT, Value: "VOI", Width: 3},
-	&config.CgrXmlCfgCdrField{Name: "FileSeqNr", Type: METATAG, Value: META_EXPORTID, Width: 5, Strip: "right", Padding: "zeroleft"},
-	&config.CgrXmlCfgCdrField{Name: "NumberOfRecords", Type: METATAG, Value: META_NRCDRS, Width: 6, Padding: "zeroleft"},
-	&config.CgrXmlCfgCdrField{Name: "CdrsDuration", Type: METATAG, Value: META_DURCDRS, Width: 8, Padding: "zeroleft"},
-	&config.CgrXmlCfgCdrField{Name: "FirstCdrTime", Type: METATAG, Value: META_FIRSTCDRTIME, Width: 12, Layout: "020106150400"},
-	&config.CgrXmlCfgCdrField{Name: "LastCdrTime", Type: METATAG, Value: META_LASTCDRTIME, Width: 12, Layout: "020106150400"},
-	&config.CgrXmlCfgCdrField{Name: "Filler2", Type: FILLER, Width: 93, Padding: "right"},
+var trailerJsnCfgFlds = []*config.CdrFieldJsonCfg{
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TypeOfRecord"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("90"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler1"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(3)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("DistributorCode"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("VOI"), Width: utils.IntPointer(3)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FileSeqNr"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_EXPORTID), Width: utils.IntPointer(5),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("zeroleft")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("NumberOfRecords"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_NRCDRS),
+		Width: utils.IntPointer(6), Padding: utils.StringPointer("zeroleft")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("CdrsDuration"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_DURCDRS),
+		Width: utils.IntPointer(8), Padding: utils.StringPointer("zeroleft"), Layout: utils.StringPointer(utils.SECONDS)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FirstCdrTime"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_FIRSTCDRATIME),
+		Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("LastCdrTime"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_LASTCDRATIME),
+		Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler2"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(93)},
 }
 
+var hdrCfgFlds, contentCfgFlds, trailerCfgFlds []*config.CfgCdrField
+
 // Write one CDR and test it's results only for content buffer
 func TestWriteCdr(t *testing.T) {
+	var err error
 	wrBuf := &bytes.Buffer{}
-	exportTpl := &config.CgrXmlCdreFwCfg{Header: &config.CgrXmlCfgCdrHeader{Fields: hdrCfgFlds},
-		Content: &config.CgrXmlCfgCdrContent{Fields: contentCfgFlds},
-		Trailer: &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds},
-	}
-	fwWriter := FixedWidthCdrWriter{writer: wrBuf, exportTemplate: exportTpl, roundDecimals: 4, header: &bytes.Buffer{}, content: &bytes.Buffer{}, trailer: &bytes.Buffer{}}
-	cdr := &utils.StoredCdr{CgrId: utils.FSCgrId("dsafdsaf"), AccId: "dsafdsaf", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
-		TOR: "call", Account: "1001", Subject: "1001", Destination: "1002",
-		SetupTime:  time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC),
-		AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC),
-		Duration:   time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.34567,
-		ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
-	}
-	if err := fwWriter.WriteCdr(cdr); err != nil {
+	cfg, _ := config.NewDefaultCGRConfig()
+	if hdrCfgFlds, err = config.CfgCdrFieldsFromCdrFieldsJsonCfg(hdrJsnCfgFlds); err != nil {
 		t.Error(err)
 	}
-	eContentOut := "201001        1001                1002                    0211  07111308420010          1       3dsafdsaf                             0002.3457 \n"
-	contentOut := fwWriter.content.String()
-	if len(contentOut) != 145 {
-		t.Error("Unexpected content length", len(contentOut))
-	} else if contentOut != eContentOut {
-		t.Errorf("Content out different than expected. Have <%s>, expecting: <%s>", contentOut, eContentOut)
+	if contentCfgFlds, err = config.CfgCdrFieldsFromCdrFieldsJsonCfg(contentJsnCfgFlds); err != nil {
+		t.Error(err)
 	}
-	eHeader := "10   VOI0000007111308420024031415390001                                                                                                         \n"
-	eTrailer := "90   VOI0000000000100000010071113084200071113084200                                                                                             \n"
-	outBeforeWrite := ""
-	if wrBuf.String() != outBeforeWrite {
-		t.Errorf("Output buffer should be empty before write")
+	if trailerCfgFlds, err = config.CfgCdrFieldsFromCdrFieldsJsonCfg(trailerJsnCfgFlds); err != nil {
+		t.Error(err)
+	}
+	cdreCfg := &config.CdreConfig{
+		CdrFormat:     utils.CDRE_FIXED_WIDTH,
+		HeaderFields:  hdrCfgFlds,
+		ContentFields: contentCfgFlds,
+		TrailerFields: trailerCfgFlds,
+	}
+	cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()),
+		TOR: utils.VOICE, OrderId: 1, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+		SetupTime:  time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC),
+		AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC),
+		Usage:      time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.34567,
+		ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
+	}
+	cdre, err := NewCdrExporter([]*engine.StoredCdr{cdr}, nil, cdreCfg, utils.CDRE_FIXED_WIDTH, ',', "fwv_1", 0.0, 0.0, 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1, cfg.HttpSkipTlsVerify)
+	if err != nil {
+		t.Error(err)
+	}
+	eHeader := "10   VOIfwv_107111308420018011511340001                                                                                                         \n"
+	eContentOut := "201001        1001                1002                    0211  07111308420010          1       3dsafdsaf                             0002.34570\n"
+	eTrailer := "90   VOIfwv_100000100000010071113084200071113084200                                                                                             \n"
+	if err := cdre.writeOut(wrBuf); err != nil {
+		t.Error(err)
 	}
-	fwWriter.Close()
 	allOut := wrBuf.String()
 	eAllOut := eHeader + eContentOut + eTrailer
 	if math.Mod(float64(len(allOut)), 145) != 0 {
@@ -113,78 +145,96 @@ func TestWriteCdr(t *testing.T) {
 		t.Errorf("Output does not match expected length. Have output %q, expecting: %q", allOut, eAllOut)
 	}
 	// Test stats
-	if !fwWriter.firstCdrTime.Equal(cdr.SetupTime) {
-		t.Error("Unexpected firstCdrTime in stats: ", fwWriter.firstCdrTime)
-	} else if !fwWriter.lastCdrTime.Equal(cdr.SetupTime) {
-		t.Error("Unexpected lastCdrTime in stats: ", fwWriter.lastCdrTime)
-	} else if fwWriter.numberOfRecords != 1 {
-		t.Error("Unexpected number of records in the stats: ", fwWriter.numberOfRecords)
-	} else if fwWriter.totalDuration != cdr.Duration {
-		t.Error("Unexpected total duration in the stats: ", fwWriter.totalDuration)
-	} else if fwWriter.totalCost != utils.Round(cdr.Cost, fwWriter.roundDecimals, utils.ROUNDING_MIDDLE) {
-		t.Error("Unexpected total cost in the stats: ", fwWriter.totalCost)
+	if !cdre.firstCdrATime.Equal(cdr.AnswerTime) {
+		t.Error("Unexpected firstCdrATime in stats: ", cdre.firstCdrATime)
+	} else if !cdre.lastCdrATime.Equal(cdr.AnswerTime) {
+		t.Error("Unexpected lastCdrATime in stats: ", cdre.lastCdrATime)
+	} else if cdre.numberOfRecords != 1 {
+		t.Error("Unexpected number of records in the stats: ", cdre.numberOfRecords)
+	} else if cdre.totalDuration != cdr.Usage {
+		t.Error("Unexpected total duration in the stats: ", cdre.totalDuration)
+	} else if cdre.totalCost != utils.Round(cdr.Cost, cdre.roundDecimals, utils.ROUNDING_MIDDLE) {
+		t.Error("Unexpected total cost in the stats: ", cdre.totalCost)
+	}
+	if cdre.FirstOrderId() != 1 {
+		t.Error("Unexpected FirstOrderId", cdre.FirstOrderId())
+	}
+	if cdre.LastOrderId() != 1 {
+		t.Error("Unexpected LastOrderId", cdre.LastOrderId())
+	}
+	if cdre.TotalCost() != utils.Round(cdr.Cost, cdre.roundDecimals, utils.ROUNDING_MIDDLE) {
+		t.Error("Unexpected TotalCost: ", cdre.TotalCost())
 	}
 }
 
 func TestWriteCdrs(t *testing.T) {
 	wrBuf := &bytes.Buffer{}
-	exportTpl := &config.CgrXmlCdreFwCfg{Header: &config.CgrXmlCfgCdrHeader{Fields: hdrCfgFlds},
-		Content: &config.CgrXmlCfgCdrContent{Fields: contentCfgFlds},
-		Trailer: &config.CgrXmlCfgCdrTrailer{Fields: trailerCfgFlds},
+	cdreCfg := &config.CdreConfig{
+		CdrFormat:     utils.CDRE_FIXED_WIDTH,
+		HeaderFields:  hdrCfgFlds,
+		ContentFields: contentCfgFlds,
+		TrailerFields: trailerCfgFlds,
 	}
-	fwWriter := FixedWidthCdrWriter{writer: wrBuf, exportTemplate: exportTpl, roundDecimals: 4, header: &bytes.Buffer{}, content: &bytes.Buffer{}, trailer: &bytes.Buffer{}}
-	cdr1 := &utils.StoredCdr{CgrId: utils.FSCgrId("aaa1"), AccId: "aaa1", CdrHost: "192.168.1.1", ReqType: "rated", Direction: "*out", Tenant: "cgrates.org",
-		TOR: "call", Account: "1001", Subject: "1001", Destination: "1010",
+	cdr1 := &engine.StoredCdr{CgrId: utils.Sha1("aaa1", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()),
+		TOR: utils.VOICE, OrderId: 2, AccId: "aaa1", CdrHost: "192.168.1.1", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1001", Subject: "1001", Destination: "1010",
 		SetupTime:  time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC),
 		AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC),
-		Duration:   time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.25,
+		Usage:      time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.25,
 		ExtraFields: map[string]string{"productnumber": "12341", "fieldextr2": "valextr2"},
 	}
-	cdr2 := &utils.StoredCdr{CgrId: utils.FSCgrId("aaa2"), AccId: "aaa2", CdrHost: "192.168.1.2", ReqType: "prepaid", Direction: "*out", Tenant: "cgrates.org",
-		TOR: "call", Account: "1002", Subject: "1002", Destination: "1011",
+	cdr2 := &engine.StoredCdr{CgrId: utils.Sha1("aaa2", time.Date(2013, 11, 7, 7, 42, 20, 0, time.UTC).String()),
+		TOR: utils.VOICE, OrderId: 4, AccId: "aaa2", CdrHost: "192.168.1.2", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1002", Subject: "1002", Destination: "1011",
 		SetupTime:  time.Date(2013, 11, 7, 7, 42, 20, 0, time.UTC),
 		AnswerTime: time.Date(2013, 11, 7, 7, 42, 26, 0, time.UTC),
-		Duration:   time.Duration(5) * time.Minute, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.40001,
+		Usage:      time.Duration(5) * time.Minute, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.40001,
 		ExtraFields: map[string]string{"productnumber": "12342", "fieldextr2": "valextr2"},
 	}
-	cdr3 := &utils.StoredCdr{}
-	cdr4 := &utils.StoredCdr{CgrId: utils.FSCgrId("aaa3"), AccId: "aaa4", CdrHost: "192.168.1.4", ReqType: "postpaid", Direction: "*out", Tenant: "cgrates.org",
-		TOR: "call", Account: "1004", Subject: "1004", Destination: "1013",
+	cdr3 := &engine.StoredCdr{}
+	cdr4 := &engine.StoredCdr{CgrId: utils.Sha1("aaa3", time.Date(2013, 11, 7, 9, 42, 18, 0, time.UTC).String()),
+		TOR: utils.VOICE, OrderId: 3, AccId: "aaa4", CdrHost: "192.168.1.4", ReqType: utils.META_POSTPAID, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1004", Subject: "1004", Destination: "1013",
 		SetupTime:  time.Date(2013, 11, 7, 9, 42, 18, 0, time.UTC),
 		AnswerTime: time.Date(2013, 11, 7, 9, 42, 26, 0, time.UTC),
-		Duration:   time.Duration(20) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.34567,
+		Usage:      time.Duration(20) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.34567,
 		ExtraFields: map[string]string{"productnumber": "12344", "fieldextr2": "valextr2"},
 	}
-	for _, cdr := range []*utils.StoredCdr{cdr1, cdr2, cdr3, cdr4} {
-		if err := fwWriter.WriteCdr(cdr); err != nil {
-			t.Error(err)
-		}
-		contentOut := fwWriter.content.String()
-		if math.Mod(float64(len(contentOut)), 145) != 0 { // Rest must be 0 always, so content is always multiple of 145 which is our row fixLength
-			t.Error("Unexpected content length", len(contentOut))
-		}
+	cfg, _ := config.NewDefaultCGRConfig()
+	cdre, err := NewCdrExporter([]*engine.StoredCdr{cdr1, cdr2, cdr3, cdr4}, nil, cdreCfg, utils.CDRE_FIXED_WIDTH, ',',
+		"fwv_1", 0.0, 0.0, 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1, cfg.HttpSkipTlsVerify)
+	if err != nil {
+		t.Error(err)
 	}
-	if len(wrBuf.String()) != 0 {
-		t.Errorf("Output buffer should be empty before write")
+	if err := cdre.writeOut(wrBuf); err != nil {
+		t.Error(err)
 	}
-	fwWriter.Close()
 	if len(wrBuf.String()) != 725 {
 		t.Error("Output buffer does not contain expected info. Expecting len: 725, got: ", len(wrBuf.String()))
 	}
 	// Test stats
-	if !fwWriter.firstCdrTime.Equal(cdr2.SetupTime) {
-		t.Error("Unexpected firstCdrTime in stats: ", fwWriter.firstCdrTime)
+	if !cdre.firstCdrATime.Equal(cdr2.AnswerTime) {
+		t.Error("Unexpected firstCdrATime in stats: ", cdre.firstCdrATime)
 	}
-	if !fwWriter.lastCdrTime.Equal(cdr4.SetupTime) {
-		t.Error("Unexpected lastCdrTime in stats: ", fwWriter.lastCdrTime)
+	if !cdre.lastCdrATime.Equal(cdr4.AnswerTime) {
+		t.Error("Unexpected lastCdrATime in stats: ", cdre.lastCdrATime)
 	}
-	if fwWriter.numberOfRecords != 3 {
-		t.Error("Unexpected number of records in the stats: ", fwWriter.numberOfRecords)
+	if cdre.numberOfRecords != 3 {
+		t.Error("Unexpected number of records in the stats: ", cdre.numberOfRecords)
 	}
-	if fwWriter.totalDuration != time.Duration(330)*time.Second {
-		t.Error("Unexpected total duration in the stats: ", fwWriter.totalDuration)
+	if cdre.totalDuration != time.Duration(330)*time.Second {
+		t.Error("Unexpected total duration in the stats: ", cdre.totalDuration)
 	}
-	if fwWriter.totalCost != 5.9957 {
-		t.Error("Unexpected total cost in the stats: ", fwWriter.totalCost)
+	if cdre.totalCost != 5.9957 {
+		t.Error("Unexpected total cost in the stats: ", cdre.totalCost)
+	}
+	if cdre.FirstOrderId() != 2 {
+		t.Error("Unexpected FirstOrderId", cdre.FirstOrderId())
+	}
+	if cdre.LastOrderId() != 4 {
+		t.Error("Unexpected LastOrderId", cdre.LastOrderId())
+	}
+	if cdre.TotalCost() != 5.9957 {
+		t.Error("Unexpected TotalCost: ", cdre.TotalCost())
 	}
 }

cdre/libcdre.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -21,6 +21,8 @@ package cdre
 import (
 	"errors"
 	"fmt"
+
+	"github.com/cgrates/cgrates/utils"
 )
 
 // Used as generic function logic for various fields
@@ -34,6 +36,9 @@ func FmtFieldWidth(source string, width int, strip, padding string, mandatory bo
 	if mandatory && len(source) == 0 {
 		return "", errors.New("Empty source value")
 	}
+	if width == 0 { // Disable width processing if not defined
+		return source, nil
+	}
 	if len(source) == width { // the source is exactly the maximum length
 		return source, nil
 	}
@@ -71,3 +76,18 @@ func FmtFieldWidth(source string, width int, strip, padding string, mandatory bo
 	}
 	return source, nil
 }
+
+// Mask a number of characters in the suffix of the destination
+func MaskDestination(dest string, maskLen int) string {
+	destLen := len(dest)
+	if maskLen < 0 {
+		return dest
+	} else if maskLen > destLen {
+		maskLen = destLen
+	}
+	dest = dest[:destLen-maskLen]
+	for i := 0; i < maskLen; i++ {
+		dest += utils.MASK_CHAR
+	}
+	return dest
+}

cdre/libcdre_test.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -114,3 +114,20 @@ func TestPaddingNotAllowed(t *testing.T) {
 		t.Error("Expected error")
 	}
 }
+
+func TestMaskDestination(t *testing.T) {
+	dest := "+4986517174963"
+	if destMasked := MaskDestination(dest, 3); destMasked != "+4986517174***" {
+		t.Error("Unexpected mask applied", destMasked)
+	}
+	if destMasked := MaskDestination(dest, -1); destMasked != dest {
+		t.Error("Negative maskLen should not modify destination", destMasked)
+	}
+	if destMasked := MaskDestination(dest, 0); destMasked != dest {
+		t.Error("Zero maskLen should not modify destination", destMasked)
+	}
+	if destMasked := MaskDestination(dest, 100); destMasked != "**************" {
+		t.Error("High maskLen should return complete mask", destMasked)
+	}
+
+}

cdrs/cdrs.go

@@ -1,93 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package cdrs
-
-import (
-	"fmt"
-	"io/ioutil"
-	"net/http"
-
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/mediator"
-	"github.com/cgrates/cgrates/utils"
-)
-
-var (
-	cfg     *config.CGRConfig // Share the configuration with the rest of the package
-	storage engine.CdrStorage
-	medi    *mediator.Mediator
-)
-
-// Returns error if not able to properly store the CDR, mediation is async since we can always recover offline
-func storeAndMediate(rawCdr utils.RawCDR) error {
-	if err := storage.SetCdr(rawCdr); err != nil {
-		return err
-	}
-	if cfg.CDRSMediator == utils.INTERNAL {
-		go func() {
-			if err := medi.RateCdr(rawCdr); err != nil {
-				engine.Logger.Err(fmt.Sprintf("Could not run mediation on CDR: %s", err.Error()))
-			}
-		}()
-	}
-	return nil
-}
-
-// Handler for generic cgr cdr http
-func cgrCdrHandler(w http.ResponseWriter, r *http.Request) {
-	cgrCdr, err := utils.NewCgrCdrFromHttpReq(r)
-	if err != nil {
-		engine.Logger.Err(fmt.Sprintf("Could not create CDR entry: %s", err.Error()))
-	}
-	if err := storeAndMediate(cgrCdr); err != nil {
-		engine.Logger.Err(fmt.Sprintf("Errors when storing CDR entry: %s", err.Error()))
-	}
-}
-
-// Handler for fs http
-func fsCdrHandler(w http.ResponseWriter, r *http.Request) {
-	body, _ := ioutil.ReadAll(r.Body)
-	fsCdr, err := new(FSCdr).New(body)
-	if err != nil {
-		engine.Logger.Err(fmt.Sprintf("Could not create CDR entry: %s", err.Error()))
-	}
-	if err := storeAndMediate(fsCdr); err != nil {
-		engine.Logger.Err(fmt.Sprintf("Errors when storing CDR entry: %s", err.Error()))
-	}
-}
-
-type CDRS struct{}
-
-func New(s engine.CdrStorage, m *mediator.Mediator, c *config.CGRConfig) *CDRS {
-	storage = s
-	medi = m
-	cfg = c
-	return &CDRS{}
-}
-
-func (cdrs *CDRS) RegisterHanlersToServer(server *engine.Server) {
-	server.RegisterHttpFunc("/cgr", cgrCdrHandler)
-	server.RegisterHttpFunc("/freeswitch_json", fsCdrHandler)
-}
-
-// Used to internally process CDR
-func (cdrs *CDRS) ProcessRawCdr(rawCdr utils.RawCDR) error {
-	return storeAndMediate(rawCdr)
-}

cdrs/fscdr.go

@@ -1,310 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package cdrs
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"reflect"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/cgrates/cgrates/engine"
-	"github.com/cgrates/cgrates/utils"
-)
-
-const (
-	// Freswitch event property names
-	FS_CDR_MAP      = "variables"
-	FS_DIRECTION    = "direction"
-	FS_SUBJECT      = "cgr_subject"
-	FS_ACCOUNT      = "cgr_account"
-	FS_DESTINATION  = "cgr_destination"
-	FS_REQTYPE      = "cgr_reqtype" //prepaid or postpaid
-	FS_TOR          = "cgr_tor"
-	FS_UUID         = "uuid" // -Unique ID for this call leg
-	FS_CSTMID       = "cgr_tenant"
-	FS_CALL_DEST_NR = "dialed_extension"
-	FS_PARK_TIME    = "start_epoch"
-	FS_SETUP_TIME   = "start_epoch"
-	FS_ANSWER_TIME  = "answer_epoch"
-	FS_HANGUP_TIME  = "end_epoch"
-	FS_DURATION     = "billsec"
-	FS_USERNAME     = "user_name"
-	FS_IP           = "sip_local_network_addr"
-	FS_CDR_SOURCE   = "freeswitch_json"
-	FS_SIP_REQUSER  = "sip_req_user" // Apps like FusionPBX do not set dialed_extension, alternative being destination_number but that comes in customer profile, not in vars
-)
-
-type FSCdr struct {
-	vars map[string]string
-	body map[string]interface{} // keeps the loaded body for extra field search
-}
-
-func (fsCdr FSCdr) New(body []byte) (utils.RawCDR, error) {
-	fsCdr.vars = make(map[string]string)
-	var err error
-	if err = json.Unmarshal(body, &fsCdr.body); err == nil {
-		if variables, ok := fsCdr.body[FS_CDR_MAP]; ok {
-			if variables, ok := variables.(map[string]interface{}); ok {
-				for k, v := range variables {
-					fsCdr.vars[k] = v.(string)
-				}
-			}
-			return fsCdr, nil
-		}
-	}
-	return nil, err
-}
-
-func (fsCdr FSCdr) GetCgrId() string {
-	return utils.FSCgrId(fsCdr.vars[FS_UUID])
-}
-func (fsCdr FSCdr) GetAccId() string {
-	return fsCdr.vars[FS_UUID]
-}
-func (fsCdr FSCdr) GetCdrHost() string {
-	return fsCdr.vars[FS_IP]
-}
-func (fsCdr FSCdr) GetCdrSource() string {
-	return FS_CDR_SOURCE
-}
-func (fsCdr FSCdr) GetDirection() string {
-	//TODO: implement direction, not related to FS_DIRECTION but traffic towards or from subject/account
-	return "*out"
-}
-func (fsCdr FSCdr) GetSubject() string {
-	return utils.FirstNonEmpty(fsCdr.vars[FS_SUBJECT], fsCdr.vars[FS_USERNAME])
-}
-func (fsCdr FSCdr) GetAccount() string {
-	return utils.FirstNonEmpty(fsCdr.vars[FS_ACCOUNT], fsCdr.vars[FS_USERNAME])
-}
-
-// Charging destination number
-func (fsCdr FSCdr) GetDestination() string {
-	return utils.FirstNonEmpty(fsCdr.vars[FS_DESTINATION], fsCdr.vars[FS_CALL_DEST_NR], fsCdr.vars[FS_SIP_REQUSER])
-}
-
-func (fsCdr FSCdr) GetTOR() string {
-	return utils.FirstNonEmpty(fsCdr.vars[FS_TOR], cfg.DefaultTOR)
-}
-
-func (fsCdr FSCdr) GetTenant() string {
-	return utils.FirstNonEmpty(fsCdr.vars[FS_CSTMID], cfg.DefaultTenant)
-}
-func (fsCdr FSCdr) GetReqType() string {
-	return utils.FirstNonEmpty(fsCdr.vars[FS_REQTYPE], cfg.DefaultReqType)
-}
-func (fsCdr FSCdr) GetExtraFields() map[string]string {
-	extraFields := make(map[string]string, len(cfg.CDRSExtraFields))
-	for _, field := range cfg.CDRSExtraFields {
-		origFieldVal, foundInVars := fsCdr.vars[field.Id]
-		if !foundInVars {
-			origFieldVal = fsCdr.searchExtraField(field.Id, fsCdr.body)
-		}
-		extraFields[field.Id] = field.ParseValue(origFieldVal)
-	}
-	return extraFields
-}
-
-func (fsCdr FSCdr) searchExtraField(field string, body map[string]interface{}) (result string) {
-	for key, value := range body {
-		switch v := value.(type) {
-		case string:
-			if key == field {
-				return v
-			}
-		case map[string]interface{}:
-			if result = fsCdr.searchExtraField(field, v); result != "" {
-				return
-			}
-		case []interface{}:
-			for _, item := range v {
-				if otherMap, ok := item.(map[string]interface{}); ok {
-					if result = fsCdr.searchExtraField(field, otherMap); result != "" {
-						return
-					}
-				} else {
-					engine.Logger.Warning(fmt.Sprintf("Slice with no maps: %v", reflect.TypeOf(item)))
-				}
-			}
-		default:
-			engine.Logger.Warning(fmt.Sprintf("Unexpected type: %v", reflect.TypeOf(v)))
-		}
-	}
-	return
-}
-
-func (fsCdr FSCdr) GetSetupTime() (t time.Time, err error) {
-	//ToDo: Make sure we work with UTC instead of local time
-	at, err := strconv.ParseInt(fsCdr.vars[FS_SETUP_TIME], 0, 64)
-	t = time.Unix(at, 0)
-	return
-}
-func (fsCdr FSCdr) GetAnswerTime() (t time.Time, err error) {
-	//ToDo: Make sure we work with UTC instead of local time
-	at, err := strconv.ParseInt(fsCdr.vars[FS_ANSWER_TIME], 0, 64)
-	t = time.Unix(at, 0)
-	return
-}
-func (fsCdr FSCdr) GetHangupTime() (t time.Time, err error) {
-	hupt, err := strconv.ParseInt(fsCdr.vars[FS_HANGUP_TIME], 0, 64)
-	t = time.Unix(hupt, 0)
-	return
-}
-
-// Extracts duration as considered by the telecom switch
-func (fsCdr FSCdr) GetDuration() time.Duration {
-	dur, _ := utils.ParseDurationWithSecs(fsCdr.vars[FS_DURATION])
-	return dur
-}
-
-func (fsCdr FSCdr) Store() (result string, err error) {
-	result += fsCdr.GetCgrId() + "|"
-	result += fsCdr.GetAccId() + "|"
-	result += fsCdr.GetCdrHost() + "|"
-	result += fsCdr.GetDirection() + "|"
-	result += fsCdr.GetSubject() + "|"
-	result += fsCdr.GetAccount() + "|"
-	result += fsCdr.GetDestination() + "|"
-	result += fsCdr.GetTOR() + "|"
-	result += fsCdr.GetAccId() + "|"
-	result += fsCdr.GetTenant() + "|"
-	result += fsCdr.GetReqType() + "|"
-	st, err := fsCdr.GetAnswerTime()
-	if err != nil {
-		return "", err
-	}
-	result += strconv.FormatInt(st.UnixNano(), 10) + "|"
-	et, err := fsCdr.GetHangupTime()
-	if err != nil {
-		return "", err
-	}
-	result += strconv.FormatInt(et.UnixNano(), 10) + "|"
-	result += strconv.FormatInt(int64(fsCdr.GetDuration().Seconds()), 10) + "|"
-	return
-}
-
-func (fsCdr FSCdr) Restore(input string) error {
-	return errors.New("Not implemented")
-}
-
-// Used in extra mediation
-func (fsCdr FSCdr) AsStoredCdr(runId, reqTypeFld, directionFld, tenantFld, torFld, accountFld, subjectFld, destFld, setupTimeFld, answerTimeFld, durationFld string, extraFlds []string, fieldsMandatory bool) (*utils.StoredCdr, error) {
-	if utils.IsSliceMember([]string{runId, reqTypeFld, directionFld, tenantFld, torFld, accountFld, subjectFld, destFld, answerTimeFld, durationFld}, "") {
-		return nil, errors.New(fmt.Sprintf("%s:FieldName", utils.ERR_MANDATORY_IE_MISSING)) // All input field names are mandatory
-	}
-	var err error
-	var hasKey bool
-	var sTimeStr, aTimeStr, durStr string
-	rtCdr := new(utils.StoredCdr)
-	rtCdr.MediationRunId = runId
-	rtCdr.Cost = -1.0 // Default for non-rated CDR
-	if rtCdr.AccId = fsCdr.GetAccId(); len(rtCdr.AccId) == 0 {
-		if fieldsMandatory {
-			return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, utils.ACCID))
-		} else { // Not mandatory, need to generate here CgrId
-			rtCdr.CgrId = utils.GenUUID()
-		}
-	} else { // hasKey, use it to generate cgrid
-		rtCdr.CgrId = utils.FSCgrId(rtCdr.AccId)
-	}
-	if rtCdr.CdrHost = fsCdr.GetCdrHost(); len(rtCdr.CdrHost) == 0 && fieldsMandatory {
-		return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, utils.CDRHOST))
-	}
-	if rtCdr.CdrSource = fsCdr.GetCdrSource(); len(rtCdr.CdrSource) == 0 && fieldsMandatory {
-		return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, utils.CDRSOURCE))
-	}
-	if strings.HasPrefix(reqTypeFld, utils.STATIC_VALUE_PREFIX) { // Values starting with prefix are not dynamically populated
-		rtCdr.ReqType = reqTypeFld[1:]
-	} else if rtCdr.ReqType, hasKey = fsCdr.vars[reqTypeFld]; !hasKey && fieldsMandatory {
-		return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, reqTypeFld))
-	}
-	if strings.HasPrefix(directionFld, utils.STATIC_VALUE_PREFIX) {
-		rtCdr.Direction = directionFld[1:]
-	} else if rtCdr.Direction, hasKey = fsCdr.vars[directionFld]; !hasKey && fieldsMandatory {
-		return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, directionFld))
-	}
-	if strings.HasPrefix(tenantFld, utils.STATIC_VALUE_PREFIX) {
-		rtCdr.Tenant = tenantFld[1:]
-	} else if rtCdr.Tenant, hasKey = fsCdr.vars[tenantFld]; !hasKey && fieldsMandatory {
-		return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, tenantFld))
-	}
-	if strings.HasPrefix(torFld, utils.STATIC_VALUE_PREFIX) {
-		rtCdr.TOR = torFld[1:]
-	} else if rtCdr.TOR, hasKey = fsCdr.vars[torFld]; !hasKey && fieldsMandatory {
-		return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, torFld))
-	}
-	if strings.HasPrefix(accountFld, utils.STATIC_VALUE_PREFIX) {
-		rtCdr.Account = accountFld[1:]
-	} else if rtCdr.Account, hasKey = fsCdr.vars[accountFld]; !hasKey && fieldsMandatory {
-		return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, accountFld))
-	}
-	if strings.HasPrefix(subjectFld, utils.STATIC_VALUE_PREFIX) {
-		rtCdr.Subject = subjectFld[1:]
-	} else if rtCdr.Subject, hasKey = fsCdr.vars[subjectFld]; !hasKey && fieldsMandatory {
-		return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, subjectFld))
-	}
-	if strings.HasPrefix(destFld, utils.STATIC_VALUE_PREFIX) {
-		rtCdr.Destination = destFld[1:]
-	} else if rtCdr.Destination, hasKey = fsCdr.vars[destFld]; !hasKey && fieldsMandatory {
-		return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, destFld))
-	}
-	if sTimeStr, hasKey = fsCdr.vars[setupTimeFld]; !hasKey && fieldsMandatory && !strings.HasPrefix(setupTimeFld, utils.STATIC_VALUE_PREFIX) {
-		return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, setupTimeFld))
-	} else {
-		if strings.HasPrefix(setupTimeFld, utils.STATIC_VALUE_PREFIX) {
-			sTimeStr = setupTimeFld[1:]
-		}
-		if rtCdr.SetupTime, err = utils.ParseTimeDetectLayout(sTimeStr); err != nil && fieldsMandatory {
-			return nil, err
-		}
-	}
-	if aTimeStr, hasKey = fsCdr.vars[answerTimeFld]; !hasKey && fieldsMandatory && !strings.HasPrefix(answerTimeFld, utils.STATIC_VALUE_PREFIX) {
-		return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, answerTimeFld))
-	} else {
-		if strings.HasPrefix(answerTimeFld, utils.STATIC_VALUE_PREFIX) {
-			aTimeStr = answerTimeFld[1:]
-		}
-		if rtCdr.AnswerTime, err = utils.ParseTimeDetectLayout(aTimeStr); err != nil && fieldsMandatory {
-			return nil, err
-		}
-	}
-	if durStr, hasKey = fsCdr.vars[durationFld]; !hasKey && fieldsMandatory && !strings.HasPrefix(durationFld, utils.STATIC_VALUE_PREFIX) {
-		return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, durationFld))
-	} else {
-		if strings.HasPrefix(durationFld, utils.STATIC_VALUE_PREFIX) {
-			durStr = durationFld[1:]
-		}
-		if rtCdr.Duration, err = utils.ParseDurationWithSecs(durStr); err != nil && fieldsMandatory {
-			return nil, err
-		}
-	}
-	rtCdr.ExtraFields = make(map[string]string, len(extraFlds))
-	for _, fldName := range extraFlds {
-		if fldVal, hasKey := fsCdr.vars[fldName]; !hasKey && fieldsMandatory {
-			return nil, errors.New(fmt.Sprintf("%s:%s", utils.ERR_MANDATORY_IE_MISSING, fldName))
-		} else {
-			rtCdr.ExtraFields[fldName] = fldVal
-		}
-	}
-	return rtCdr, nil
-}

cmd/cgr-console/cgr-console.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -22,51 +22,161 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
+	"log"
+	"os"
+	"strings"
 
 	"github.com/cgrates/cgrates/console"
 	"github.com/cgrates/cgrates/utils"
-
-	"log"
-	"net/rpc"
-	"net/rpc/jsonrpc"
-	"os"
+	"github.com/cgrates/liner"
+	"github.com/cgrates/rpcclient"
 )
 
 var (
+	history_fn   = os.Getenv("HOME") + "/.cgr_history"
 	version      = flag.Bool("version", false, "Prints the application version.")
+	verbose      = flag.Bool("verbose", false, "Show extra info about command execution.")
 	server       = flag.String("server", "127.0.0.1:2012", "server address host:port")
 	rpc_encoding = flag.String("rpc_encoding", "json", "RPC encoding used <gob|json>")
+	client       *rpcclient.RpcClient
 )
 
+func executeCommand(command string) {
+	if strings.TrimSpace(command) == "" {
+		return
+	}
+	if strings.TrimSpace(command) == "help" {
+		commands := console.GetCommands()
+		fmt.Println("Commands:")
+		for name, cmd := range commands {
+			fmt.Print(name, cmd.Usage())
+		}
+		return
+	}
+	if strings.HasPrefix(command, "help") {
+		words := strings.Split(command, " ")
+		if len(words) > 1 {
+			commands := console.GetCommands()
+			if cmd, ok := commands[words[1]]; ok {
+				fmt.Print(cmd.Usage())
+			} else {
+				fmt.Print("Available commands: ")
+				for name, _ := range commands {
+					fmt.Print(name + " ")
+				}
+				fmt.Println()
+			}
+			return
+		}
+	}
+	cmd, cmdErr := console.GetCommandValue(command, *verbose)
+	if cmdErr != nil {
+		fmt.Println(cmdErr)
+		return
+	}
+	if cmd.RpcMethod() != "" {
+		res := cmd.RpcResult()
+		param := cmd.RpcParams(false)
+		//log.Print(reflect.TypeOf(param))
+		switch param.(type) {
+		case *console.EmptyWrapper:
+			param = ""
+		case *console.StringWrapper:
+			param = param.(*console.StringWrapper).Item
+		case *console.StringSliceWrapper:
+			param = param.(*console.StringSliceWrapper).Items
+		case *console.StringMapWrapper:
+			param = param.(*console.StringMapWrapper).Items
+		}
+		//log.Printf("Param: %+v", param)
+
+		if rpcErr := client.Call(cmd.RpcMethod(), param, res); rpcErr != nil {
+			fmt.Println("Error executing command: " + rpcErr.Error())
+		} else {
+			result, _ := json.MarshalIndent(res, "", " ")
+			fmt.Println(string(result))
+		}
+	} else {
+		fmt.Println(cmd.LocalExecute())
+	}
+}
+
 func main() {
 	flag.Parse()
 	if *version {
 		fmt.Println("CGRateS " + utils.VERSION)
 		return
 	}
-	var client *rpc.Client
 	var err error
-	if *rpc_encoding == "json" {
-		client, err = jsonrpc.Dial("tcp", *server)
-	} else {
-		client, err = rpc.Dial("tcp", *server)
-	}
+	client, err = rpcclient.NewRpcClient("tcp", *server, 3, 3, *rpc_encoding)
 	if err != nil {
 		flag.PrintDefaults()
 		log.Fatal("Could not connect to server " + *server)
 	}
-	defer client.Close()
-	// Strict command parsing starts here
-	args := append([]string{os.Args[0]}, flag.Args()...) // Emulate os.Args by prepending the cmd to list of args coming from flag
-	cmd, cmdErr := console.GetCommandValue(args)
-	if cmdErr != nil {
-		log.Fatal(cmdErr)
-	}
-	res := cmd.RpcResult()
-	if rpcErr := client.Call(cmd.RpcMethod(), cmd.RpcParams(), res); rpcErr != nil {
-		fmt.Println("Error executing command: " + rpcErr.Error())
-	}
-	result, _ := json.MarshalIndent(res, "", "  ")
-	fmt.Println(string(result))
 
+	if len(flag.Args()) != 0 {
+		executeCommand(strings.Join(flag.Args(), " "))
+		return
+	}
+
+	fmt.Println("Welcome to CGRateS console!")
+	fmt.Print("Type `help` for a list of commands\n\n")
+
+	line := liner.NewLiner()
+	defer line.Close()
+
+	line.SetCompleter(func(line string) (comp []string) {
+		commands := console.GetCommands()
+		for name, cmd := range commands {
+			if strings.HasPrefix(name, strings.ToLower(line)) {
+				comp = append(comp, name)
+			}
+			// try arguments
+			if strings.HasPrefix(line, name) {
+				// get last word
+				lastSpace := strings.LastIndex(line, " ")
+				lastSpace += 1
+				for _, arg := range cmd.ClientArgs() {
+					if strings.HasPrefix(arg, line[lastSpace:]) {
+						comp = append(comp, line[:lastSpace]+arg)
+					}
+				}
+			}
+		}
+		return
+	})
+
+	if f, err := os.Open(history_fn); err == nil {
+		line.ReadHistory(f)
+		f.Close()
+	}
+
+	stop := false
+	for !stop {
+		if command, err := line.Prompt("cgr> "); err != nil {
+			if err == io.EOF {
+				fmt.Println("\nbye!")
+				stop = true
+			} else {
+				fmt.Print("Error reading line: ", err)
+			}
+		} else {
+			line.AppendHistory(command)
+			switch strings.ToLower(strings.TrimSpace(command)) {
+			case "quit", "exit", "bye", "close":
+				fmt.Println("\nbye!")
+				stop = true
+			default:
+				executeCommand(command)
+			}
+		}
+	}
+
+	if f, err := os.Create(history_fn); err != nil {
+		log.Print("Error writing history file: ", err)
+	} else {
+		line.WriteHistory(f)
+		f.Close()
+	}
 }

cmd/cgr-engine/cgr-engine.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -19,27 +19,27 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package main
 
 import (
-	"errors"
 	"flag"
 	"fmt"
 	"log"
-	"net/rpc"
 	"os"
 	"runtime"
+	"runtime/pprof"
 	"strconv"
+	"sync"
 	"time"
 
-	"github.com/cgrates/cgrates/apier"
+	"github.com/cgrates/cgrates/apier/v1"
+	"github.com/cgrates/cgrates/apier/v2"
 	"github.com/cgrates/cgrates/balancer2go"
 	"github.com/cgrates/cgrates/cdrc"
-	"github.com/cgrates/cgrates/cdrs"
 	"github.com/cgrates/cgrates/config"
 	"github.com/cgrates/cgrates/engine"
 	"github.com/cgrates/cgrates/history"
-	"github.com/cgrates/cgrates/mediator"
 	"github.com/cgrates/cgrates/scheduler"
 	"github.com/cgrates/cgrates/sessionmanager"
 	"github.com/cgrates/cgrates/utils"
+	"github.com/cgrates/rpcclient"
 )
 
 const (
@@ -51,83 +51,62 @@ const (
 	REDIS    = "redis"
 	SAME     = "same"
 	FS       = "freeswitch"
+	KAMAILIO = "kamailio"
+	OSIPS    = "opensips"
 )
 
 var (
-	cfgPath         = flag.String("config", "/etc/cgrates/cgrates.cfg", "Configuration file location.")
-	version         = flag.Bool("version", false, "Prints the application version.")
-	raterEnabled    = flag.Bool("rater", false, "Enforce starting of the rater daemon overwriting config")
-	schedEnabled    = flag.Bool("scheduler", false, "Enforce starting of the scheduler daemon .overwriting config")
-	cdrsEnabled     = flag.Bool("cdrs", false, "Enforce starting of the cdrs daemon overwriting config")
-	cdrcEnabled     = flag.Bool("cdrc", false, "Enforce starting of the cdrc service overwriting config")
-	mediatorEnabled = flag.Bool("mediator", false, "Enforce starting of the mediator service overwriting config")
-	pidFile         = flag.String("pid", "", "Write pid file")
-	bal             = balancer2go.NewBalancer()
-	exitChan        = make(chan bool)
-	server          = &engine.Server{}
-	scribeServer    history.Scribe
-	cdrServer       *cdrs.CDRS
-	sm              sessionmanager.SessionManager
-	medi            *mediator.Mediator
-	cfg             *config.CGRConfig
-	err             error
+	cfgDir       = flag.String("config_dir", utils.CONFIG_DIR, "Configuration directory path.")
+	version      = flag.Bool("version", false, "Prints the application version.")
+	raterEnabled = flag.Bool("rater", false, "Enforce starting of the rater daemon overwriting config")
+	schedEnabled = flag.Bool("scheduler", false, "Enforce starting of the scheduler daemon .overwriting config")
+	cdrsEnabled  = flag.Bool("cdrs", false, "Enforce starting of the cdrs daemon overwriting config")
+	pidFile      = flag.String("pid", "", "Write pid file")
+	cpuprofile   = flag.String("cpuprofile", "", "write cpu profile to file")
+	singlecpu    = flag.Bool("singlecpu", false, "Run on single CPU core")
+	bal          = balancer2go.NewBalancer()
+	exitChan     = make(chan bool)
+	server       = &engine.Server{}
+	cdrServer    *engine.CdrServer
+	cdrStats     engine.StatsInterface
+	scribeServer history.Scribe
+	pubSubServer engine.PublisherSubscriber
+	userServer   engine.UserService
+	cfg          *config.CGRConfig
+	sms          []sessionmanager.SessionManager
+	smRpc        *v1.SessionManagerV1
+	err          error
 )
 
 func cacheData(ratingDb engine.RatingStorage, accountDb engine.AccountingStorage, doneChan chan struct{}) {
-	if err := ratingDb.CacheRating(nil, nil, nil, nil); err != nil {
+	if err := ratingDb.CacheAll(); err != nil {
 		engine.Logger.Crit(fmt.Sprintf("Cache rating error: %s", err.Error()))
 		exitChan <- true
 		return
 	}
-	if err := accountDb.CacheAccounting(nil, nil, nil); err != nil {
-		engine.Logger.Crit(fmt.Sprintf("Cache accounting error: %s", err.Error()))
-		exitChan <- true
-		return
-	}
 	close(doneChan)
 }
 
-func startMediator(responder *engine.Responder, loggerDb engine.LogStorage, cdrDb engine.CdrStorage, cacheChan, chanDone chan struct{}) {
-	var connector engine.Connector
-	if cfg.MediatorRater == utils.INTERNAL {
-		<-cacheChan // Cache needs to come up before we are ready
-		connector = responder
+// Fires up a cdrc instance
+func startCdrc(responder *engine.Responder, cdrsChan chan struct{}, cdrcCfgs map[string]*config.CdrcConfig, httpSkipTlsCheck bool, closeChan chan struct{}) {
+	var cdrsConn engine.Connector
+	var cdrcCfg *config.CdrcConfig
+	for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one
+		break
+	}
+	if cdrcCfg.Cdrs == utils.INTERNAL {
+		<-cdrsChan // Wait for CDRServer to come up before start processing
+		cdrsConn = responder
 	} else {
-		var client *rpc.Client
-		var err error
-
-		for i := 0; i < cfg.MediatorRaterReconnects; i++ {
-			client, err = rpc.Dial("tcp", cfg.MediatorRater)
-			if err == nil { //Connected so no need to reiterate
-				break
-			}
-			time.Sleep(time.Duration(i+1) * time.Second)
-		}
+		conn, err := rpcclient.NewRpcClient("tcp", cdrcCfg.Cdrs, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
 		if err != nil {
-			engine.Logger.Crit(fmt.Sprintf("<Mediator> Could not connect to engine: %v", err))
+			engine.Logger.Crit(fmt.Sprintf("<CDRC> Could not connect to CDRS via RPC: %v", err))
 			exitChan <- true
 			return
 		}
-		connector = &engine.RPCClientConnector{Client: client}
+		cdrsConn = &engine.RPCClientConnector{Client: conn}
 	}
-	var err error
-	medi, err = mediator.NewMediator(connector, loggerDb, cdrDb, cfg)
-	if err != nil {
-		engine.Logger.Crit(fmt.Sprintf("Mediator config parsing error: %v", err))
-		exitChan <- true
-		return
-	}
-	engine.Logger.Info("Registering Mediator RPC service.")
-	server.RpcRegister(&mediator.MediatorV1{Medi: medi})
-
-	close(chanDone)
-}
-
-func startCdrc(cdrsChan chan struct{}) {
-	if cfg.CdrcCdrs == utils.INTERNAL {
-		<-cdrsChan // Wait for CDRServer to come up before start processing
-	}
-	cdrc, err := cdrc.NewCdrc(cfg, cdrServer)
+	cdrc, err := cdrc.NewCdrc(cdrcCfgs, httpSkipTlsCheck, cdrsConn, closeChan)
 	if err != nil {
 		engine.Logger.Crit(fmt.Sprintf("Cdrc config parsing error: %s", err.Error()))
 		exitChan <- true
@@ -139,93 +118,222 @@ func startCdrc(cdrsChan chan struct{}) {
 	exitChan <- true // If run stopped, something is bad, stop the application
 }
 
-func startSessionManager(responder *engine.Responder, loggerDb engine.LogStorage, cacheChan chan struct{}) {
-	var connector engine.Connector
-	if cfg.SMRater == utils.INTERNAL {
+func startSmFreeSWITCH(responder *engine.Responder, cdrDb engine.CdrStorage, cacheChan chan struct{}) {
+	var raterConn, cdrsConn engine.Connector
+	var client *rpcclient.RpcClient
+	delay := utils.Fib()
+	if cfg.SmFsConfig.Rater == utils.INTERNAL {
 		<-cacheChan // Wait for the cache to init before start doing queries
-		connector = responder
+		raterConn = responder
 	} else {
-		var client *rpc.Client
 		var err error
-
-		for i := 0; i < cfg.SMRaterReconnects; i++ {
-			client, err = rpc.Dial("tcp", cfg.SMRater)
+		for i := 0; i < cfg.SmFsConfig.Reconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.SmFsConfig.Rater, cfg.ConnectAttempts, cfg.SmFsConfig.Reconnects, utils.GOB)
 			if err == nil { //Connected so no need to reiterate
 				break
 			}
-			time.Sleep(time.Duration(i+1) * time.Second)
+			time.Sleep(delay())
 		}
 		if err != nil {
-			engine.Logger.Crit(fmt.Sprintf("<SessionManager> Could not connect to engine: %v", err))
+			engine.Logger.Crit(fmt.Sprintf("<SM-FreeSWITCH> Could not connect to rater via RPC: %v", err))
 			exitChan <- true
+			return
 		}
-		connector = &engine.RPCClientConnector{Client: client}
+		raterConn = &engine.RPCClientConnector{Client: client}
 	}
-	switch cfg.SMSwitchType {
-	case FS:
-		dp, _ := time.ParseDuration(fmt.Sprintf("%vs", cfg.SMDebitInterval))
-		sm = sessionmanager.NewFSSessionManager(loggerDb, connector, dp)
-		errConn := sm.Connect(cfg)
-		if errConn != nil {
-			engine.Logger.Err(fmt.Sprintf("<SessionManager> error: %s!", errConn))
+	if cfg.SmFsConfig.Cdrs == cfg.SmFsConfig.Rater {
+		cdrsConn = raterConn
+	} else if cfg.SmFsConfig.Cdrs == utils.INTERNAL {
+		<-cacheChan // Wait for the cache to init before start doing queries
+		cdrsConn = responder
+	} else if len(cfg.SmFsConfig.Cdrs) != 0 {
+		delay = utils.Fib()
+		for i := 0; i < cfg.SmFsConfig.Reconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.SmFsConfig.Cdrs, cfg.ConnectAttempts, cfg.SmFsConfig.Reconnects, utils.GOB)
+			if err == nil { //Connected so no need to reiterate
+				break
+			}
+			time.Sleep(delay())
 		}
-	default:
-		engine.Logger.Err(fmt.Sprintf("<SessionManager> Unsupported session manger type: %s!", cfg.SMSwitchType))
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<SM-FreeSWITCH> Could not connect to CDRS via RPC: %v", err))
+			exitChan <- true
+			return
+		}
+		cdrsConn = &engine.RPCClientConnector{Client: client}
+	}
+	sm := sessionmanager.NewFSSessionManager(cfg.SmFsConfig, raterConn, cdrsConn)
+	sms = append(sms, sm)
+	smRpc.SMs = append(smRpc.SMs, sm)
+	if err = sm.Connect(); err != nil {
+		engine.Logger.Err(fmt.Sprintf("<SessionManager> error: %s!", err))
 	}
 	exitChan <- true
 }
 
-func startCDRS(responder *engine.Responder, cdrDb engine.CdrStorage, mediChan, doneChan chan struct{}) {
-	if cfg.CDRSMediator == utils.INTERNAL {
-		<-mediChan // Deadlock if mediator not started
-		if medi == nil {
-			engine.Logger.Crit("<CDRS> Could not connect to mediator, exiting.")
+func startSmKamailio(responder *engine.Responder, cdrDb engine.CdrStorage, cacheChan chan struct{}) {
+	var raterConn, cdrsConn engine.Connector
+	var client *rpcclient.RpcClient
+	if cfg.SmKamConfig.Rater == utils.INTERNAL {
+		<-cacheChan // Wait for the cache to init before start doing queries
+		raterConn = responder
+	} else {
+		var err error
+		delay := utils.Fib()
+		for i := 0; i < cfg.SmKamConfig.Reconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.SmKamConfig.Rater, cfg.ConnectAttempts, cfg.SmKamConfig.Reconnects, utils.GOB)
+			if err == nil { //Connected so no need to reiterate
+				break
+			}
+			time.Sleep(delay())
+		}
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<SessionManager> Could not connect to rater: %v", err))
+			exitChan <- true
+		}
+		raterConn = &engine.RPCClientConnector{Client: client}
+	}
+	if cfg.SmKamConfig.Cdrs == cfg.SmKamConfig.Rater {
+		cdrsConn = raterConn
+	} else if cfg.SmKamConfig.Cdrs == utils.INTERNAL {
+		<-cacheChan // Wait for the cache to init before start doing queries
+		cdrsConn = responder
+	} else if len(cfg.SmKamConfig.Cdrs) != 0 {
+		delay := utils.Fib()
+		for i := 0; i < cfg.SmKamConfig.Reconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.SmKamConfig.Cdrs, cfg.ConnectAttempts, cfg.SmKamConfig.Reconnects, utils.GOB)
+			if err == nil { //Connected so no need to reiterate
+				break
+			}
+			time.Sleep(delay())
+		}
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<SM-Kamailio> Could not connect to CDRS via RPC: %v", err))
 			exitChan <- true
 			return
 		}
+		cdrsConn = &engine.RPCClientConnector{Client: client}
 	}
-	cdrServer = cdrs.New(cdrDb, medi, cfg)
-	cdrServer.RegisterHanlersToServer(server)
-	close(doneChan)
+	sm, _ := sessionmanager.NewKamailioSessionManager(cfg.SmKamConfig, raterConn, cdrsConn)
+	sms = append(sms, sm)
+	smRpc.SMs = append(smRpc.SMs, sm)
+	if err = sm.Connect(); err != nil {
+		engine.Logger.Err(fmt.Sprintf("<SessionManager> error: %s!", err))
+	}
+	exitChan <- true
 }
 
-func startHistoryServer(chanDone chan struct{}) {
-	if scribeServer, err = history.NewFileScribe(cfg.HistoryDir, cfg.HistorySaveInterval); err != nil {
-		engine.Logger.Crit(fmt.Sprintf("<HistoryServer> Could not start, error: %s", err.Error()))
-		exitChan <- true
-		return
+func startSmOpenSIPS(responder *engine.Responder, cdrDb engine.CdrStorage, cacheChan chan struct{}) {
+	var raterConn, cdrsConn engine.Connector
+	var client *rpcclient.RpcClient
+	if cfg.SmOsipsConfig.Rater == utils.INTERNAL {
+		<-cacheChan // Wait for the cache to init before start doing queries
+		raterConn = responder
+	} else {
+		var err error
+		delay := utils.Fib()
+		for i := 0; i < cfg.SmOsipsConfig.Reconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.SmOsipsConfig.Rater, cfg.ConnectAttempts, cfg.SmOsipsConfig.Reconnects, utils.GOB)
+			if err == nil { //Connected so no need to reiterate
+				break
+			}
+			time.Sleep(delay())
+		}
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<SessionManager> Could not connect to rater: %v", err))
+			exitChan <- true
+		}
+		raterConn = &engine.RPCClientConnector{Client: client}
 	}
-	server.RpcRegisterName("Scribe", scribeServer)
-	close(chanDone)
-}
-
-// chanStartServer will report when server is up, useful for internal requests
-func startHistoryAgent(scribeServer history.Scribe, chanServerStarted chan struct{}) {
-	if cfg.HistoryServer == utils.INTERNAL { // For internal requests, wait for server to come online before connecting
-		engine.Logger.Crit(fmt.Sprintf("<HistoryAgent> Connecting internally to HistoryServer"))
-		select {
-		case <-time.After(1 * time.Minute):
-			engine.Logger.Crit(fmt.Sprintf("<HistoryAgent> Timeout waiting for server to start."))
+	if cfg.SmOsipsConfig.Cdrs == cfg.SmOsipsConfig.Rater {
+		cdrsConn = raterConn
+	} else if cfg.SmOsipsConfig.Cdrs == utils.INTERNAL {
+		<-cacheChan // Wait for the cache to init before start doing queries
+		cdrsConn = responder
+	} else if len(cfg.SmOsipsConfig.Cdrs) != 0 {
+		delay := utils.Fib()
+		for i := 0; i < cfg.SmOsipsConfig.Reconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.SmOsipsConfig.Cdrs, cfg.ConnectAttempts, cfg.SmOsipsConfig.Reconnects, utils.GOB)
+			if err == nil { //Connected so no need to reiterate
+				break
+			}
+			time.Sleep(delay())
+		}
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<SM-OpenSIPS> Could not connect to CDRS via RPC: %v", err))
 			exitChan <- true
 			return
-		case <-chanServerStarted:
 		}
-		//<-chanServerStarted // If server is not enabled, will have deadlock here
-	} else { // Connect in iteration since there are chances of concurrency here
-		for i := 0; i < 3; i++ { //ToDo: Make it globally configurable
-			//engine.Logger.Crit(fmt.Sprintf("<HistoryAgent> Trying to connect, iteration: %d, time %s", i, time.Now()))
-			if scribeServer, err = history.NewProxyScribe(cfg.HistoryServer); err == nil {
-				break //Connected so no need to reiterate
-			} else if i == 2 && err != nil {
-				engine.Logger.Crit(fmt.Sprintf("<HistoryAgent> Could not connect to the server, error: %s", err.Error()))
+		cdrsConn = &engine.RPCClientConnector{Client: client}
+	}
+	sm, _ := sessionmanager.NewOSipsSessionManager(cfg.SmOsipsConfig, raterConn, cdrsConn)
+	sms = append(sms, sm)
+	smRpc.SMs = append(smRpc.SMs, sm)
+	if err := sm.Connect(); err != nil {
+		engine.Logger.Err(fmt.Sprintf("<SM-OpenSIPS> error: %s!", err))
+	}
+	exitChan <- true
+}
+
+func startCDRS(logDb engine.LogStorage, cdrDb engine.CdrStorage, responder *engine.Responder, responderReady, doneChan chan struct{}) {
+	var err error
+	var client *rpcclient.RpcClient
+	// Rater connection init
+	var raterConn engine.Connector
+	if cfg.CDRSRater == utils.INTERNAL {
+		<-responderReady // Wait for the cache to init before start doing queries
+		raterConn = responder
+	} else if len(cfg.CDRSRater) != 0 {
+		delay := utils.Fib()
+		for i := 0; i < cfg.CDRSReconnects; i++ {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSRater, cfg.ConnectAttempts, cfg.CDRSReconnects, utils.GOB)
+			if err == nil { //Connected so no need to reiterate
+				break
+			}
+			time.Sleep(delay())
+		}
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<CDRS> Could not connect to rater: %s", err.Error()))
+			exitChan <- true
+			return
+		}
+		raterConn = &engine.RPCClientConnector{Client: client}
+	}
+	// Stats connection init
+	var statsConn engine.StatsInterface
+	if cfg.CDRSStats == utils.INTERNAL {
+		statsConn = cdrStats
+	} else if len(cfg.CDRSStats) != 0 {
+		if cfg.CDRSRater == cfg.CDRSStats {
+			statsConn = &engine.ProxyStats{Client: client}
+		} else {
+			delay := utils.Fib()
+			for i := 0; i < cfg.CDRSReconnects; i++ {
+				client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSStats, cfg.ConnectAttempts, cfg.CDRSReconnects, utils.GOB)
+				if err == nil { //Connected so no need to reiterate
+					break
+				}
+				time.Sleep(delay())
+			}
+			if err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<CDRS> Could not connect to stats server: %s", err.Error()))
 				exitChan <- true
 				return
 			}
-			time.Sleep(time.Duration(i) * time.Second)
+			statsConn = &engine.ProxyStats{Client: client}
 		}
 	}
-	engine.SetHistoryScribe(scribeServer)
-	return
+
+	cdrServer, _ = engine.NewCdrServer(cfg, cdrDb, raterConn, statsConn)
+	engine.Logger.Info("Registering CDRS HTTP Handlers.")
+	cdrServer.RegisterHanlersToServer(server)
+	engine.Logger.Info("Registering CDRS RPC service.")
+	cdrSrv := v1.CdrsV1{CdrSrv: cdrServer}
+	server.RpcRegister(&cdrSrv)
+	server.RpcRegister(&v2.CdrsV2{CdrsV1: cdrSrv})
+	// Make the cdr servers available for internal communication
+	responder.CdrSrv = cdrServer
+	close(doneChan)
 }
 
 // Starts the rpc server, waiting for the necessary components to finish their tasks
@@ -246,26 +354,6 @@ func serveHttp(httpWaitChans []chan struct{}) {
 	server.ServeHTTP(cfg.HTTPListen)
 }
 
-func checkConfigSanity() error {
-	if cfg.SMEnabled && cfg.RaterEnabled && cfg.RaterBalancer != "" {
-		engine.Logger.Crit("The session manager must not be enabled on a worker engine (change [engine]/balancer to disabled)!")
-		return errors.New("SessionManager on Worker")
-	}
-	if cfg.BalancerEnabled && cfg.RaterEnabled && cfg.RaterBalancer != "" {
-		engine.Logger.Crit("The balancer is enabled so it cannot connect to another balancer (change rater/balancer to disabled)!")
-		return errors.New("Improperly configured balancer")
-	}
-	if cfg.CDRSEnabled && cfg.CDRSMediator == utils.INTERNAL && !cfg.MediatorEnabled {
-		engine.Logger.Crit("CDRS cannot connect to mediator, Mediator not enabled in configuration!")
-		return errors.New("Internal Mediator required by CDRS")
-	}
-	if cfg.HistoryServerEnabled && cfg.HistoryServer == utils.INTERNAL && !cfg.HistoryServerEnabled {
-		engine.Logger.Crit("The history agent is enabled and internal and history server is disabled!")
-		return errors.New("Improperly configured history service")
-	}
-	return nil
-}
-
 func writePid() {
 	engine.Logger.Info(*pidFile)
 	f, err := os.Create(*pidFile)
@@ -287,9 +375,18 @@ func main() {
 	if *pidFile != "" {
 		writePid()
 	}
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
-	cfg, err = config.NewCGRConfig(cfgPath)
+	if !*singlecpu {
+		runtime.GOMAXPROCS(runtime.NumCPU()) // For now it slows down computing due to CPU management, to be reviewed in future Go releases
+	}
+	if *cpuprofile != "" {
+		f, err := os.Create(*cpuprofile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+	cfg, err = config.NewCGRConfigFromFolder(*cfgDir)
 	if err != nil {
 		engine.Logger.Crit(fmt.Sprintf("Could not parse config: %s exiting!", err))
 		return
@@ -304,64 +401,49 @@ func main() {
 	if *cdrsEnabled {
 		cfg.CDRSEnabled = *cdrsEnabled
 	}
-	if *cdrcEnabled {
-		cfg.CdrcEnabled = *cdrcEnabled
-	}
-	if *mediatorEnabled {
-		cfg.MediatorEnabled = *mediatorEnabled
-	}
-
-	// some consitency checks
-	errCfg := checkConfigSanity()
-	if errCfg != nil {
-		engine.Logger.Crit(errCfg.Error())
-		return
-	}
 	var ratingDb engine.RatingStorage
 	var accountDb engine.AccountingStorage
 	var logDb engine.LogStorage
 	var loadDb engine.LoadStorage
 	var cdrDb engine.CdrStorage
-	ratingDb, err = engine.ConfigureRatingStorage(cfg.RatingDBType, cfg.RatingDBHost, cfg.RatingDBPort,
-		cfg.RatingDBName, cfg.RatingDBUser, cfg.RatingDBPass, cfg.DBDataEncoding)
-	if err != nil { // Cannot configure getter database, show stopper
-		engine.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err))
-		return
-	}
-	defer ratingDb.Close()
-	engine.SetRatingStorage(ratingDb)
-	accountDb, err = engine.ConfigureAccountingStorage(cfg.AccountDBType, cfg.AccountDBHost, cfg.AccountDBPort,
-		cfg.AccountDBName, cfg.AccountDBUser, cfg.AccountDBPass, cfg.DBDataEncoding)
-	if err != nil { // Cannot configure getter database, show stopper
-		engine.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err))
-		return
-	}
-	defer accountDb.Close()
-	engine.SetAccountingStorage(accountDb)
-
-	if cfg.StorDBType == SAME {
-		logDb = ratingDb.(engine.LogStorage)
-	} else {
-		logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort,
-			cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding)
-		if err != nil { // Cannot configure logger database, show stopper
-			engine.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting!", err))
+	if cfg.RaterEnabled || cfg.SchedulerEnabled { // Only connect to dataDb if required
+		ratingDb, err = engine.ConfigureRatingStorage(cfg.TpDbType, cfg.TpDbHost, cfg.TpDbPort,
+			cfg.TpDbName, cfg.TpDbUser, cfg.TpDbPass, cfg.DBDataEncoding)
+		if err != nil { // Cannot configure getter database, show stopper
+			engine.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err))
 			return
 		}
-	}
-	defer logDb.Close()
-	engine.SetStorageLogger(logDb)
-	// loadDb,cdrDb and logDb are all mapped on the same stordb storage
-	loadDb = logDb.(engine.LoadStorage)
-	cdrDb = logDb.(engine.CdrStorage)
-
-	engine.SetRoundingMethodAndDecimals(cfg.RoundingMethod, cfg.RoundingDecimals)
-	if cfg.SMDebitInterval > 0 {
-		if dp, err := time.ParseDuration(fmt.Sprintf("%vs", cfg.SMDebitInterval)); err == nil {
-			engine.SetDebitPeriod(dp)
+		defer ratingDb.Close()
+		engine.SetRatingStorage(ratingDb)
+		accountDb, err = engine.ConfigureAccountingStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort,
+			cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding)
+		if err != nil { // Cannot configure getter database, show stopper
+			engine.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err))
+			return
 		}
+		defer accountDb.Close()
+		engine.SetAccountingStorage(accountDb)
+	}
+	if cfg.RaterEnabled || cfg.CDRSEnabled || cfg.SchedulerEnabled { // Only connect to storDb if necessary
+		if cfg.StorDBType == SAME {
+			logDb = ratingDb.(engine.LogStorage)
+		} else {
+			logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort,
+				cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns)
+			if err != nil { // Cannot configure logger database, show stopper
+				engine.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting!", err))
+				return
+			}
+		}
+		defer logDb.Close()
+		engine.SetStorageLogger(logDb)
+		// loadDb,cdrDb and logDb are all mapped on the same stordb storage
+		loadDb = logDb.(engine.LoadStorage)
+		cdrDb = logDb.(engine.CdrStorage)
+		engine.SetCdrStorage(cdrDb)
 	}
 
+	engine.SetRoundingDecimals(cfg.RoundingDecimals)
 	stopHandled := false
 
 	// Async starts here
@@ -381,14 +463,101 @@ func main() {
 		go stopRaterSignalHandler()
 		stopHandled = true
 	}
+	if cfg.CDRStatsEnabled { // Init it here so we make it availabe to the Apier
+		cdrStats = engine.NewStats(ratingDb, accountDb, cfg.CDRStatsSaveInterval)
+		server.RpcRegister(cdrStats)
+		server.RpcRegister(&v1.CDRStatsV1{CdrStats: cdrStats}) // Public APIs
+	}
 
-	responder := &engine.Responder{ExitChan: exitChan}
-	apier := &apier.ApierV1{StorDb: loadDb, RatingDb: ratingDb, AccountDb: accountDb, CdrDb: cdrDb, LogDb: logDb, Config: cfg}
+	if cfg.PubSubServerEnabled {
+		pubSubServer = engine.NewPubSub(accountDb, cfg.HttpSkipTlsVerify)
+		server.RpcRegisterName("PubSubV1", pubSubServer)
+	}
+
+	if cfg.HistoryServerEnabled {
+		scribeServer, err = history.NewFileScribe(cfg.HistoryDir, cfg.HistorySaveInterval)
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<HistoryServer> Could not start, error: %s", err.Error()))
+			exitChan <- true
+		}
+		server.RpcRegisterName("ScribeV1", scribeServer)
+	}
+	if cfg.UserServerEnabled {
+		userServer, err = engine.NewUserMap(accountDb, cfg.UserServerIndexes)
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<UsersService> Could not start, error: %s", err.Error()))
+			exitChan <- true
+		}
+		server.RpcRegisterName("UsersV1", userServer)
+	}
+
+	// Register session manager service // FixMe: make sure this is thread safe
+	if cfg.SmFsConfig.Enabled || cfg.SmKamConfig.Enabled || cfg.SmOsipsConfig.Enabled { // Register SessionManagerV1 service
+		smRpc = new(v1.SessionManagerV1)
+		server.RpcRegister(smRpc)
+	}
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if cfg.RaterCdrStats != "" && cfg.RaterCdrStats != utils.INTERNAL {
+			if cdrStats, err = engine.NewProxyStats(cfg.RaterCdrStats, cfg.ConnectAttempts, -1); err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<CdrStats> Could not connect to the server, error: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if cfg.RaterHistoryServer != "" && cfg.RaterHistoryServer != utils.INTERNAL {
+			if scribeServer, err = history.NewProxyScribe(cfg.RaterHistoryServer, cfg.ConnectAttempts, -1); err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<HistoryServer> Could not connect to the server, error: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+		}
+		engine.SetHistoryScribe(scribeServer)
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if cfg.RaterPubSubServer != "" && cfg.RaterPubSubServer != utils.INTERNAL {
+			if pubSubServer, err = engine.NewProxyPubSub(cfg.RaterPubSubServer, cfg.ConnectAttempts, -1); err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<PubSubServer> Could not connect to the server, error: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+		}
+		engine.SetPubSub(pubSubServer)
+	}()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if cfg.RaterUserServer != "" && cfg.RaterUserServer != utils.INTERNAL {
+			if userServer, err = engine.NewProxyUserService(cfg.RaterUserServer, cfg.ConnectAttempts, -1); err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<UserServer> Could not connect to the server, error: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+		}
+		engine.SetUserService(userServer)
+	}()
+	wg.Wait()
+
+	responder := &engine.Responder{ExitChan: exitChan, Stats: cdrStats}
+	apierRpcV1 := &v1.ApierV1{StorDb: loadDb, RatingDb: ratingDb, AccountDb: accountDb, CdrDb: cdrDb, LogDb: logDb, Config: cfg, Responder: responder, CdrStatsSrv: cdrStats, Users: userServer}
+	apierRpcV2 := &v2.ApierV2{
+		ApierV1: v1.ApierV1{StorDb: loadDb, RatingDb: ratingDb, AccountDb: accountDb, CdrDb: cdrDb, LogDb: logDb, Config: cfg, Responder: responder, CdrStatsSrv: cdrStats, Users: userServer}}
 
 	if cfg.RaterEnabled && !cfg.BalancerEnabled && cfg.RaterBalancer != utils.INTERNAL {
 		engine.Logger.Info("Registering Rater service")
 		server.RpcRegister(responder)
-		server.RpcRegister(apier)
+		server.RpcRegister(apierRpcV1)
+		server.RpcRegister(apierRpcV2)
 	}
 
 	if cfg.BalancerEnabled {
@@ -397,7 +566,8 @@ func main() {
 		stopHandled = true
 		responder.Bal = bal
 		server.RpcRegister(responder)
-		server.RpcRegister(apier)
+		server.RpcRegister(apierRpcV1)
+		server.RpcRegister(apierRpcV2)
 		if cfg.RaterEnabled {
 			engine.Logger.Info("<Balancer> Registering internal rater")
 			bal.AddClient("local", new(engine.ResponderWorker))
@@ -412,56 +582,56 @@ func main() {
 		engine.Logger.Info("Starting CGRateS Scheduler.")
 		go func() {
 			sched := scheduler.NewScheduler()
-			go reloadSchedulerSingnalHandler(sched, accountDb)
-			apier.Sched = sched
-			sched.LoadActionTimings(accountDb)
+			go reloadSchedulerSingnalHandler(sched, ratingDb)
+			apierRpcV1.Sched = sched
+			apierRpcV2.Sched = sched
+			sched.LoadActionPlans(ratingDb)
 			sched.Loop()
 		}()
 	}
 
-	var histServChan chan struct{} // Will be initialized only if the server starts
-	if cfg.HistoryServerEnabled {
-		histServChan = make(chan struct{})
-		rpcWait = append(rpcWait, histServChan)
-		go startHistoryServer(histServChan)
-	}
-
-	if cfg.HistoryAgentEnabled {
-		engine.Logger.Info("Starting CGRateS History Agent.")
-		go startHistoryAgent(scribeServer, histServChan)
-	}
-
-	var medChan chan struct{}
-	if cfg.MediatorEnabled {
-		engine.Logger.Info("Starting CGRateS Mediator service.")
-		medChan = make(chan struct{})
-		go startMediator(responder, logDb, cdrDb, cacheChan, medChan)
-	}
-
 	var cdrsChan chan struct{}
 	if cfg.CDRSEnabled {
 		engine.Logger.Info("Starting CGRateS CDRS service.")
 		cdrsChan = make(chan struct{})
 		httpWait = append(httpWait, cdrsChan)
-		go startCDRS(responder, cdrDb, medChan, cdrsChan)
+		go startCDRS(logDb, cdrDb, responder, cacheChan, cdrsChan)
 	}
 
-	if cfg.SMEnabled {
-		engine.Logger.Info("Starting CGRateS SessionManager service.")
-		go startSessionManager(responder, logDb, cacheChan)
+	if cfg.SmFsConfig.Enabled {
+		engine.Logger.Info("Starting CGRateS SM-FreeSWITCH service.")
+		go startSmFreeSWITCH(responder, cdrDb, cacheChan)
 		// close all sessions on shutdown
 		go shutdownSessionmanagerSingnalHandler()
 	}
-
-	if cfg.CdrcEnabled {
+	if cfg.SmKamConfig.Enabled {
+		engine.Logger.Info("Starting CGRateS SM-Kamailio service.")
+		go startSmKamailio(responder, cdrDb, cacheChan)
+	}
+	if cfg.SmOsipsConfig.Enabled {
+		engine.Logger.Info("Starting CGRateS SM-OpenSIPS service.")
+		go startSmOpenSIPS(responder, cdrDb, cacheChan)
+	}
+	var cdrcEnabled bool
+	for _, cdrcCfgs := range cfg.CdrcProfiles {
+		var cdrcCfg *config.CdrcConfig
+		for _, cdrcCfg = range cdrcCfgs { // Take a random config out since they should be the same
+			break
+		}
+		if cdrcCfg.Enabled == false {
+			continue // Ignore not enabled
+		} else if !cdrcEnabled {
+			cdrcEnabled = true // Mark that at least one cdrc service is active
+		}
+		go startCdrc(responder, cdrsChan, cdrcCfgs, cfg.HttpSkipTlsVerify, cfg.ConfigReloads[utils.CDRC])
+	}
+	if cdrcEnabled {
 		engine.Logger.Info("Starting CGRateS CDR client.")
-		go startCdrc(cdrsChan)
 	}
 
 	// Start the servers
 	go serveRpc(rpcWait)
 	go serveHttp(httpWait)
-
 	<-exitChan
 
 	if *pidFile != "" {

cmd/cgr-engine/engine_test.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

cmd/cgr-engine/registration.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -48,6 +48,10 @@ func generalSignalHandler() {
 
 	sig := <-c
 	engine.Logger.Info(fmt.Sprintf("Caught signal %v, shuting down cgr-engine\n", sig))
+	var dummyInt int
+	if cdrStats != nil {
+		cdrStats.Stop(dummyInt, &dummyInt)
+	}
 	exitChan <- true
 }
 
@@ -61,6 +65,8 @@ func stopRaterSignalHandler() {
 
 	engine.Logger.Info(fmt.Sprintf("Caught signal %v, unregistering from balancer\n", sig))
 	unregisterFromBalancer()
+	var dummyInt int
+	cdrStats.Stop(dummyInt, &dummyInt)
 	exitChan <- true
 }
 
@@ -104,14 +110,14 @@ func registerToBalancer() {
 }
 
 // Listens for the HUP system signal and gracefuly reloads the timers from database.
-func reloadSchedulerSingnalHandler(sched *scheduler.Scheduler, getter engine.AccountingStorage) {
+func reloadSchedulerSingnalHandler(sched *scheduler.Scheduler, getter engine.RatingStorage) {
 	for {
 		c := make(chan os.Signal)
 		signal.Notify(c, syscall.SIGHUP)
 		sig := <-c
 
 		engine.Logger.Info(fmt.Sprintf("Caught signal %v, reloading action timings.\n", sig))
-		sched.LoadActionTimings(getter)
+		sched.LoadActionPlans(getter)
 		// check the tip of the queue for new actions
 		sched.Restart()
 	}
@@ -125,8 +131,10 @@ func shutdownSessionmanagerSingnalHandler() {
 	signal.Notify(c, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT)
 	<-c
 
-	if err := sm.Shutdown(); err != nil {
-		engine.Logger.Warning(fmt.Sprintf("<SessionManager> %s", err))
+	for _, sm := range sms {
+		if err := sm.Shutdown(); err != nil {
+			engine.Logger.Warning(fmt.Sprintf("<SessionManager> %s", err))
+		}
 	}
 	exitChan <- true
 }

cmd/cgr-loader/cgr-loader.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -34,19 +34,19 @@ import (
 var (
 	//separator = flag.String("separator", ",", "Default field separator")
 	cgrConfig, _  = config.NewDefaultCGRConfig()
-	ratingdb_type = flag.String("ratingdb_type", cgrConfig.RatingDBType, "The type of the RatingDb database <redis>")
-	ratingdb_host = flag.String("ratingdb_host", cgrConfig.RatingDBHost, "The RatingDb host to connect to.")
-	ratingdb_port = flag.String("ratingdb_port", cgrConfig.RatingDBPort, "The RatingDb port to bind to.")
-	ratingdb_name = flag.String("ratingdb_name", cgrConfig.RatingDBName, "The name/number of the RatingDb to connect to.")
-	ratingdb_user = flag.String("ratingdb_user", cgrConfig.RatingDBUser, "The RatingDb user to sign in as.")
-	ratingdb_pass = flag.String("ratingdb_passwd", cgrConfig.RatingDBPass, "The RatingDb user's password.")
+	ratingdb_type = flag.String("ratingdb_type", cgrConfig.TpDbType, "The type of the RatingDb database <redis>")
+	ratingdb_host = flag.String("ratingdb_host", cgrConfig.TpDbHost, "The RatingDb host to connect to.")
+	ratingdb_port = flag.String("ratingdb_port", cgrConfig.TpDbPort, "The RatingDb port to bind to.")
+	ratingdb_name = flag.String("ratingdb_name", cgrConfig.TpDbName, "The name/number of the RatingDb to connect to.")
+	ratingdb_user = flag.String("ratingdb_user", cgrConfig.TpDbUser, "The RatingDb user to sign in as.")
+	ratingdb_pass = flag.String("ratingdb_passwd", cgrConfig.TpDbPass, "The RatingDb user's password.")
 
-	accountdb_type = flag.String("accountdb_type", cgrConfig.AccountDBType, "The type of the AccountingDb database <redis>")
-	accountdb_host = flag.String("accountdb_host", cgrConfig.AccountDBHost, "The AccountingDb host to connect to.")
-	accountdb_port = flag.String("accountdb_port", cgrConfig.AccountDBPort, "The AccountingDb port to bind to.")
-	accountdb_name = flag.String("accountdb_name", cgrConfig.AccountDBName, "The name/number of the AccountingDb to connect to.")
-	accountdb_user = flag.String("accountdb_user", cgrConfig.AccountDBUser, "The AccountingDb user to sign in as.")
-	accountdb_pass = flag.String("accountdb_passwd", cgrConfig.AccountDBPass, "The AccountingDb user's password.")
+	accountdb_type = flag.String("accountdb_type", cgrConfig.DataDbType, "The type of the AccountingDb database <redis>")
+	accountdb_host = flag.String("accountdb_host", cgrConfig.DataDbHost, "The AccountingDb host to connect to.")
+	accountdb_port = flag.String("accountdb_port", cgrConfig.DataDbPort, "The AccountingDb port to bind to.")
+	accountdb_name = flag.String("accountdb_name", cgrConfig.DataDbName, "The name/number of the AccountingDb to connect to.")
+	accountdb_user = flag.String("accountdb_user", cgrConfig.DataDbUser, "The AccountingDb user to sign in as.")
+	accountdb_pass = flag.String("accountdb_passwd", cgrConfig.DataDbPass, "The AccountingDb user's password.")
 
 	stor_db_type = flag.String("stordb_type", cgrConfig.StorDBType, "The type of the storDb database <mysql>")
 	stor_db_host = flag.String("stordb_host", cgrConfig.StorDBHost, "The storDb host to connect to.")
@@ -57,18 +57,21 @@ var (
 
 	dbdata_encoding = flag.String("dbdata_encoding", cgrConfig.DBDataEncoding, "The encoding used to store object data in strings")
 
-	flush         = flag.Bool("flushdb", false, "Flush the database before importing")
-	tpid          = flag.String("tpid", "", "The tariff plan id from the database")
-	dataPath      = flag.String("path", "./", "The path to folder containing the data files")
-	version       = flag.Bool("version", false, "Prints the application version.")
-	verbose       = flag.Bool("verbose", false, "Enable detailed verbose logging output")
-	dryRun        = flag.Bool("dry_run", false, "When true will not save loaded data to dataDb but just parse it for consistency and errors.")
-	stats         = flag.Bool("stats", false, "Generates statsistics about given data.")
-	fromStorDb    = flag.Bool("from_stordb", false, "Load the tariff plan from storDb to dataDb")
-	toStorDb      = flag.Bool("to_stordb", false, "Import the tariff plan from files to storDb")
-	historyServer = flag.String("history_server", cgrConfig.RPCGOBListen, "The history server address:port, empty to disable automaticautomatic  history archiving")
-	raterAddress  = flag.String("rater_address", cgrConfig.RPCGOBListen, "Rater service to contact for cache reloads, empty to disable automatic cache reloads")
-	runId         = flag.String("runid", "", "Uniquely identify an import/load, postpended to some automatic fields")
+	flush           = flag.Bool("flushdb", false, "Flush the database before importing")
+	tpid            = flag.String("tpid", "", "The tariff plan id from the database")
+	dataPath        = flag.String("path", "./", "The path to folder containing the data files")
+	version         = flag.Bool("version", false, "Prints the application version.")
+	verbose         = flag.Bool("verbose", false, "Enable detailed verbose logging output")
+	dryRun          = flag.Bool("dry_run", false, "When true will not save loaded data to dataDb but just parse it for consistency and errors.")
+	validate        = flag.Bool("validate", false, "When true will run various check on the loaded data to check for structural errors")
+	stats           = flag.Bool("stats", false, "Generates statsistics about given data.")
+	fromStorDb      = flag.Bool("from_stordb", false, "Load the tariff plan from storDb to dataDb")
+	toStorDb        = flag.Bool("to_stordb", false, "Import the tariff plan from files to storDb")
+	historyServer   = flag.String("history_server", cgrConfig.RPCGOBListen, "The history server address:port, empty to disable automaticautomatic  history archiving")
+	raterAddress    = flag.String("rater_address", cgrConfig.RPCGOBListen, "Rater service to contact for cache reloads, empty to disable automatic cache reloads")
+	cdrstatsAddress = flag.String("cdrstats_address", cgrConfig.RPCGOBListen, "CDRStats service to contact for data reloads, empty to disable automatic data reloads")
+	usersAddress    = flag.String("users_address", cgrConfig.RPCGOBListen, "Users service to contact for data reloads, empty to disable automatic data reloads")
+	runId           = flag.String("runid", "", "Uniquely identify an import/load, postpended to some automatic fields")
 )
 
 func main() {
@@ -81,17 +84,19 @@ func main() {
 	var ratingDb engine.RatingStorage
 	var accountDb engine.AccountingStorage
 	var storDb engine.LoadStorage
-	var rater *rpc.Client
-	var loader engine.TPLoader
+	var rater, cdrstats, users *rpc.Client
+	var loader engine.LoadReader
 	// Init necessary db connections, only if not already
 	if !*dryRun { // make sure we do not need db connections on dry run, also not importing into any stordb
 		if *fromStorDb {
 			ratingDb, errRatingDb = engine.ConfigureRatingStorage(*ratingdb_type, *ratingdb_host, *ratingdb_port, *ratingdb_name,
 				*ratingdb_user, *ratingdb_pass, *dbdata_encoding)
 			accountDb, errAccDb = engine.ConfigureAccountingStorage(*accountdb_type, *accountdb_host, *accountdb_port, *accountdb_name, *accountdb_user, *accountdb_pass, *dbdata_encoding)
-			storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding)
+			storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding,
+				cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns)
 		} else if *toStorDb { // Import from csv files to storDb
-			storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding)
+			storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding,
+				cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns)
 		} else { // Default load from csv files to dataDb
 			ratingDb, errRatingDb = engine.ConfigureRatingStorage(*ratingdb_type, *ratingdb_host, *ratingdb_port, *ratingdb_name,
 				*ratingdb_user, *ratingdb_pass, *dbdata_encoding)
@@ -113,7 +118,14 @@ func main() {
 			if *tpid == "" {
 				log.Fatal("TPid required, please define it via *-tpid* command argument.")
 			}
-			csvImporter := engine.TPCSVImporter{*tpid, storDb, *dataPath, ',', *verbose, *runId}
+			csvImporter := engine.TPCSVImporter{
+				TPid:     *tpid,
+				StorDb:   storDb,
+				DirPath:  *dataPath,
+				Sep:      ',',
+				Verbose:  *verbose,
+				ImportId: *runId,
+			}
 			if errImport := csvImporter.Run(); errImport != nil {
 				log.Fatal(errImport)
 			}
@@ -121,15 +133,15 @@ func main() {
 		}
 	}
 	if *fromStorDb { // Load Tariff Plan from storDb into dataDb
-		loader = engine.NewDbReader(storDb, ratingDb, accountDb, *tpid)
+		loader = storDb
 	} else { // Default load from csv files to dataDb
-		for fn, v := range engine.FileValidators {
+		/*for fn, v := range engine.FileValidators {
 			err := engine.ValidateCSVData(path.Join(*dataPath, fn), v.Rule)
 			if err != nil {
 				log.Fatal(err, "\n\t", v.Message)
 			}
-		}
-		loader = engine.NewFileCSVReader(ratingDb, accountDb, ',',
+		}*/
+		loader = engine.NewFileCSVStorage(',',
 			path.Join(*dataPath, utils.DESTINATIONS_CSV),
 			path.Join(*dataPath, utils.TIMINGS_CSV),
 			path.Join(*dataPath, utils.RATES_CSV),
@@ -137,28 +149,38 @@ func main() {
 			path.Join(*dataPath, utils.RATING_PLANS_CSV),
 			path.Join(*dataPath, utils.RATING_PROFILES_CSV),
 			path.Join(*dataPath, utils.SHARED_GROUPS_CSV),
+			path.Join(*dataPath, utils.LCRS_CSV),
 			path.Join(*dataPath, utils.ACTIONS_CSV),
 			path.Join(*dataPath, utils.ACTION_PLANS_CSV),
 			path.Join(*dataPath, utils.ACTION_TRIGGERS_CSV),
-			path.Join(*dataPath, utils.ACCOUNT_ACTIONS_CSV))
+			path.Join(*dataPath, utils.ACCOUNT_ACTIONS_CSV),
+			path.Join(*dataPath, utils.DERIVED_CHARGERS_CSV),
+			path.Join(*dataPath, utils.CDR_STATS_CSV),
+			path.Join(*dataPath, utils.USERS_CSV))
 	}
-	err = loader.LoadAll()
+	tpReader := engine.NewTpReader(ratingDb, accountDb, loader, *tpid)
+	err = tpReader.LoadAll()
 	if err != nil {
 		log.Fatal(err)
 	}
 	if *stats {
-		loader.ShowStatistics()
+		tpReader.ShowStatistics()
+	}
+	if *validate {
+		if !tpReader.IsValid() {
+			return
+		}
 	}
 	if *dryRun { // We were just asked to parse the data, not saving it
 		return
 	}
 	if *historyServer != "" { // Init scribeAgent so we can store the differences
-		if scribeAgent, err := history.NewProxyScribe(*historyServer); err != nil {
+		if scribeAgent, err := history.NewProxyScribe(*historyServer, 3, 3); err != nil {
 			log.Fatalf("Could not connect to history server, error: %s. Make sure you have properly configured it via -history_server flag.", err.Error())
 			return
 		} else {
 			engine.SetHistoryScribe(scribeAgent)
-			defer scribeAgent.Client.Close()
+			//defer scribeAgent.Client.Close()
 		}
 	} else {
 		log.Print("WARNING: Rates history archiving is disabled!")
@@ -172,9 +194,35 @@ func main() {
 	} else {
 		log.Print("WARNING: Rates automatic cache reloading is disabled!")
 	}
+	if *cdrstatsAddress != "" { // Init connection to rater so we can reload it's data
+		if *cdrstatsAddress == *raterAddress {
+			cdrstats = rater
+		} else {
+			cdrstats, err = rpc.Dial("tcp", *cdrstatsAddress)
+			if err != nil {
+				log.Fatalf("Could not connect to CDRStats API: %s", err.Error())
+				return
+			}
+		}
+	} else {
+		log.Print("WARNING: CDRStats automatic data reload is disabled!")
+	}
+	if *usersAddress != "" { // Init connection to rater so we can reload it's data
+		if *usersAddress == *raterAddress {
+			users = rater
+		} else {
+			users, err = rpc.Dial("tcp", *usersAddress)
+			if err != nil {
+				log.Fatalf("Could not connect to Users API: %s", err.Error())
+				return
+			}
+		}
+	} else {
+		log.Print("WARNING: Users automatic data reload is disabled!")
+	}
 
 	// write maps to database
-	if err := loader.WriteToDatabase(*flush, *verbose); err != nil {
+	if err := tpReader.WriteToDatabase(*flush, *verbose); err != nil {
 		log.Fatal("Could not write to database: ", err)
 	}
 	if len(*historyServer) != 0 && *verbose {
@@ -183,29 +231,73 @@ func main() {
 	// Reload scheduler and cache
 	if rater != nil {
 		reply := ""
-		dstIds, _ := loader.GetLoadedIds(engine.DESTINATION_PREFIX)
-		rplIds, _ := loader.GetLoadedIds(engine.RATING_PLAN_PREFIX)
-		rpfIds, _ := loader.GetLoadedIds(engine.RATING_PROFILE_PREFIX)
-		actIds, _ := loader.GetLoadedIds(engine.ACTION_PREFIX)
-		shgIds, _ := loader.GetLoadedIds(engine.SHARED_GROUP_PREFIX)
-		rpAliases, _ := loader.GetLoadedIds(engine.RP_ALIAS_PREFIX)
-		accAliases, _ := loader.GetLoadedIds(engine.ACC_ALIAS_PREFIX)
+		dstIds, _ := tpReader.GetLoadedIds(utils.DESTINATION_PREFIX)
+		rplIds, _ := tpReader.GetLoadedIds(utils.RATING_PLAN_PREFIX)
+		rpfIds, _ := tpReader.GetLoadedIds(utils.RATING_PROFILE_PREFIX)
+		actIds, _ := tpReader.GetLoadedIds(utils.ACTION_PREFIX)
+		shgIds, _ := tpReader.GetLoadedIds(utils.SHARED_GROUP_PREFIX)
+		rpAliases, _ := tpReader.GetLoadedIds(utils.RP_ALIAS_PREFIX)
+		accAliases, _ := tpReader.GetLoadedIds(utils.ACC_ALIAS_PREFIX)
+		lcrIds, _ := tpReader.GetLoadedIds(utils.LCR_PREFIX)
+		dcs, _ := tpReader.GetLoadedIds(utils.DERIVEDCHARGERS_PREFIX)
 		// Reload cache first since actions could be calling info from within
 		if *verbose {
 			log.Print("Reloading cache")
 		}
-		if err = rater.Call("ApierV1.ReloadCache", utils.ApiReloadCache{dstIds, rplIds, rpfIds, actIds, shgIds, rpAliases, accAliases}, &reply); err != nil {
-			log.Fatalf("Got error on cache reload: %s", err.Error())
+		if *flush {
+			dstIds, rplIds, rpfIds, rpAliases, lcrIds = nil, nil, nil, nil, nil // Should reload all these on flush
 		}
-		actTmgIds, _ := loader.GetLoadedIds(engine.ACTION_TIMING_PREFIX)
+		if err = rater.Call("ApierV1.ReloadCache", utils.ApiReloadCache{
+			DestinationIds:   dstIds,
+			RatingPlanIds:    rplIds,
+			RatingProfileIds: rpfIds,
+			ActionIds:        actIds,
+			SharedGroupIds:   shgIds,
+			RpAliases:        rpAliases,
+			AccAliases:       accAliases,
+			LCRIds:           lcrIds,
+			DerivedChargers:  dcs,
+		}, &reply); err != nil {
+			log.Printf("WARNING: Got error on cache reload: %s\n", err.Error())
+		}
+		actTmgIds, _ := tpReader.GetLoadedIds(utils.ACTION_TIMING_PREFIX)
 		if len(actTmgIds) != 0 {
 			if *verbose {
 				log.Print("Reloading scheduler")
 			}
 			if err = rater.Call("ApierV1.ReloadScheduler", "", &reply); err != nil {
-				log.Fatalf("Got error on scheduler reload: %s", err.Error())
+				log.Printf("WARNING: Got error on scheduler reload: %s\n", err.Error())
 			}
 		}
 
 	}
+	if cdrstats != nil {
+		statsQueueIds, _ := tpReader.GetLoadedIds(utils.CDR_STATS_PREFIX)
+		if *flush {
+			statsQueueIds = []string{} // Force reload all
+		}
+		if len(statsQueueIds) != 0 {
+			if *verbose {
+				log.Print("Reloading CDRStats data")
+			}
+			var reply string
+			if err := cdrstats.Call("CDRStatsV1.ReloadQueues", utils.AttrCDRStatsReloadQueues{StatsQueueIds: statsQueueIds}, &reply); err != nil {
+				log.Printf("WARNING: Failed reloading stat queues, error: %s\n", err.Error())
+			}
+		}
+	}
+
+	if users != nil {
+		userIds, _ := tpReader.GetLoadedIds(utils.USERS_PREFIX)
+		if len(userIds) > 0 {
+			if *verbose {
+				log.Print("Reloading Users data")
+			}
+			var reply string
+			if err := cdrstats.Call("UsersV1.ReloadUsers", "", &reply); err != nil {
+				log.Printf("WARNING: Failed reloading users data, error: %s\n", err.Error())
+			}
+
+		}
+	}
 }

cmd/cgr-tester/cgr-tester.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"log"
 	"net/rpc"
+	"net/rpc/jsonrpc"
 	"os"
 	"runtime"
 	"runtime/pprof"
@@ -30,6 +31,7 @@ import (
 
 	"github.com/cgrates/cgrates/config"
 	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
 )
 
 var (
@@ -38,24 +40,26 @@ var (
 	memprofile      = flag.String("memprofile", "", "write memory profile to this file")
 	runs            = flag.Int("runs", 10000, "stress cycle number")
 	parallel        = flag.Int("parallel", 0, "run n requests in parallel")
-	ratingdb_type   = flag.String("ratingdb_type", cgrConfig.RatingDBType, "The type of the RatingDb database <redis>")
-	ratingdb_host   = flag.String("ratingdb_host", cgrConfig.RatingDBHost, "The RatingDb host to connect to.")
-	ratingdb_port   = flag.String("ratingdb_port", cgrConfig.RatingDBPort, "The RatingDb port to bind to.")
-	ratingdb_name   = flag.String("ratingdb_name", cgrConfig.RatingDBName, "The name/number of the RatingDb to connect to.")
-	ratingdb_user   = flag.String("ratingdb_user", cgrConfig.RatingDBUser, "The RatingDb user to sign in as.")
-	ratingdb_pass   = flag.String("ratingdb_passwd", cgrConfig.RatingDBPass, "The RatingDb user's password.")
-	accountdb_type  = flag.String("accountdb_type", cgrConfig.AccountDBType, "The type of the AccountingDb database <redis>")
-	accountdb_host  = flag.String("accountdb_host", cgrConfig.AccountDBHost, "The AccountingDb host to connect to.")
-	accountdb_port  = flag.String("accountdb_port", cgrConfig.AccountDBPort, "The AccountingDb port to bind to.")
-	accountdb_name  = flag.String("accountdb_name", cgrConfig.AccountDBName, "The name/number of the AccountingDb to connect to.")
-	accountdb_user  = flag.String("accountdb_user", cgrConfig.AccountDBUser, "The AccountingDb user to sign in as.")
-	accountdb_pass  = flag.String("accountdb_passwd", cgrConfig.AccountDBPass, "The AccountingDb user's password.")
+	ratingdb_type   = flag.String("ratingdb_type", cgrConfig.TpDbType, "The type of the RatingDb database <redis>")
+	ratingdb_host   = flag.String("ratingdb_host", cgrConfig.TpDbHost, "The RatingDb host to connect to.")
+	ratingdb_port   = flag.String("ratingdb_port", cgrConfig.TpDbPort, "The RatingDb port to bind to.")
+	ratingdb_name   = flag.String("ratingdb_name", cgrConfig.TpDbName, "The name/number of the RatingDb to connect to.")
+	ratingdb_user   = flag.String("ratingdb_user", cgrConfig.TpDbUser, "The RatingDb user to sign in as.")
+	ratingdb_pass   = flag.String("ratingdb_passwd", cgrConfig.TpDbPass, "The RatingDb user's password.")
+	accountdb_type  = flag.String("accountdb_type", cgrConfig.DataDbType, "The type of the AccountingDb database <redis>")
+	accountdb_host  = flag.String("accountdb_host", cgrConfig.DataDbHost, "The AccountingDb host to connect to.")
+	accountdb_port  = flag.String("accountdb_port", cgrConfig.DataDbPort, "The AccountingDb port to bind to.")
+	accountdb_name  = flag.String("accountdb_name", cgrConfig.DataDbName, "The name/number of the AccountingDb to connect to.")
+	accountdb_user  = flag.String("accountdb_user", cgrConfig.DataDbUser, "The AccountingDb user to sign in as.")
+	accountdb_pass  = flag.String("accountdb_passwd", cgrConfig.DataDbPass, "The AccountingDb user's password.")
 	dbdata_encoding = flag.String("dbdata_encoding", cgrConfig.DBDataEncoding, "The encoding used to store object data in strings.")
 	raterAddress    = flag.String("rater_address", "", "Rater address for remote tests. Empty for internal rater.")
-	tor             = flag.String("tor", "call", "The type of record to use in queries.")
-	tenant          = flag.String("tenant", "call", "The type of record to use in queries.")
+	tor             = flag.String("tor", utils.VOICE, "The type of record to use in queries.")
+	category        = flag.String("category", "call", "The Record category to test.")
+	tenant          = flag.String("tenant", "cgrates.org", "The type of record to use in queries.")
 	subject         = flag.String("subject", "1001", "The rating subject to use in queries.")
-	destination     = flag.String("destination", "+4986517174963", "The destination to use in queries.")
+	destination     = flag.String("destination", "1002", "The destination to use in queries.")
+	json            = flag.Bool("json", false, "Use JSON RPC")
 
 	nilDuration = time.Duration(0)
 )
@@ -73,7 +77,7 @@ func durInternalRater(cd *engine.CallDescriptor) (time.Duration, error) {
 	}
 	defer accountDb.Close()
 	engine.SetAccountingStorage(accountDb)
-	if err := ratingDb.CacheRating(nil, nil, nil, nil); err != nil {
+	if err := ratingDb.CacheAll(); err != nil {
 		return nilDuration, fmt.Errorf("Cache rating error: %s", err.Error())
 	}
 	log.Printf("Runnning %d cycles...", *runs)
@@ -105,9 +109,16 @@ func durInternalRater(cd *engine.CallDescriptor) (time.Duration, error) {
 
 func durRemoteRater(cd *engine.CallDescriptor) (time.Duration, error) {
 	result := engine.CallCost{}
-	client, err := rpc.Dial("tcp", *raterAddress)
+	var client *rpc.Client
+	var err error
+	if *json {
+		client, err = jsonrpc.Dial("tcp", *raterAddress)
+	} else {
+		client, err = rpc.Dial("tcp", *raterAddress)
+	}
+
 	if err != nil {
-		return nilDuration, fmt.Errorf("Could not connect to engine: ", err.Error())
+		return nilDuration, fmt.Errorf("Could not connect to engine: %s", err.Error())
 	}
 	defer client.Close()
 	start := time.Now()
@@ -150,14 +161,15 @@ func main() {
 		defer pprof.StopCPUProfile()
 	}
 	cd := &engine.CallDescriptor{
-		TimeStart:    time.Date(2013, time.December, 13, 22, 30, 0, 0, time.UTC),
-		TimeEnd:      time.Date(2013, time.December, 13, 22, 31, 0, 0, time.UTC),
-		CallDuration: 60 * time.Second,
-		Direction:    "*out",
-		TOR:          *tor,
-		Tenant:       *tenant,
-		Subject:      *subject,
-		Destination:  *destination,
+		TimeStart:     time.Date(2014, time.December, 11, 55, 30, 0, 0, time.UTC),
+		TimeEnd:       time.Date(2014, time.December, 11, 55, 31, 0, 0, time.UTC),
+		DurationIndex: 60 * time.Second,
+		Direction:     "*out",
+		TOR:           *tor,
+		Category:      *category,
+		Tenant:        *tenant,
+		Subject:       *subject,
+		Destination:   *destination,
 	}
 	var duration time.Duration
 	var err error

config/cdrcconfig.go

@@ -0,0 +1,146 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"time"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+type CdrcConfig struct {
+	Enabled                 bool            // Enable/Disable the profile
+	DryRun                  bool            // Do not post CDRs to the server
+	Cdrs                    string          // The address where CDRs can be reached
+	CdrFormat               string          // The type of CDR file to process <csv|opensips_flatstore>
+	FieldSeparator          rune            // The separator to use when reading csvs
+	DataUsageMultiplyFactor float64         // Conversion factor for data usage
+	RunDelay                time.Duration   // Delay between runs, 0 for inotify driven requests
+	MaxOpenFiles            int             // Maximum number of files opened simultaneously
+	CdrInDir                string          // Folder to process CDRs from
+	CdrOutDir               string          // Folder to move processed CDRs to
+	FailedCallsPrefix       string          // Used in case of flatstore CDRs to avoid searching for BYE records
+	CdrSourceId             string          // Source identifier for the processed CDRs
+	CdrFilter               utils.RSRFields // Filter CDR records to import
+	PartialRecordCache      time.Duration   // Duration to cache partial records when not pairing
+	HeaderFields            []*CfgCdrField
+	ContentFields           []*CfgCdrField
+	TrailerFields           []*CfgCdrField
+}
+
+func (self *CdrcConfig) loadFromJsonCfg(jsnCfg *CdrcJsonCfg) error {
+	if jsnCfg == nil {
+		return nil
+	}
+	var err error
+	if jsnCfg.Enabled != nil {
+		self.Enabled = *jsnCfg.Enabled
+	}
+	if jsnCfg.Dry_run != nil {
+		self.DryRun = *jsnCfg.Dry_run
+	}
+	if jsnCfg.Cdrs != nil {
+		self.Cdrs = *jsnCfg.Cdrs
+	}
+	if jsnCfg.Cdr_format != nil {
+		self.CdrFormat = *jsnCfg.Cdr_format
+	}
+	if jsnCfg.Field_separator != nil && len(*jsnCfg.Field_separator) > 0 {
+		sepStr := *jsnCfg.Field_separator
+		self.FieldSeparator = rune(sepStr[0])
+	}
+	if jsnCfg.Data_usage_multiply_factor != nil {
+		self.DataUsageMultiplyFactor = *jsnCfg.Data_usage_multiply_factor
+	}
+	if jsnCfg.Run_delay != nil {
+		self.RunDelay = time.Duration(*jsnCfg.Run_delay) * time.Second
+	}
+	if jsnCfg.Max_open_files != nil {
+		self.MaxOpenFiles = *jsnCfg.Max_open_files
+	}
+	if jsnCfg.Cdr_in_dir != nil {
+		self.CdrInDir = *jsnCfg.Cdr_in_dir
+	}
+	if jsnCfg.Cdr_out_dir != nil {
+		self.CdrOutDir = *jsnCfg.Cdr_out_dir
+	}
+	if jsnCfg.Failed_calls_prefix != nil {
+		self.FailedCallsPrefix = *jsnCfg.Failed_calls_prefix
+	}
+	if jsnCfg.Cdr_source_id != nil {
+		self.CdrSourceId = *jsnCfg.Cdr_source_id
+	}
+	if jsnCfg.Cdr_filter != nil {
+		if self.CdrFilter, err = utils.ParseRSRFields(*jsnCfg.Cdr_filter, utils.INFIELD_SEP); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Partial_record_cache != nil {
+		if self.PartialRecordCache, err = utils.ParseDurationWithSecs(*jsnCfg.Partial_record_cache); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Header_fields != nil {
+		if self.HeaderFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Header_fields); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Content_fields != nil {
+		if self.ContentFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Content_fields); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Trailer_fields != nil {
+		if self.TrailerFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Trailer_fields); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Clone itself into a new CdrcConfig
+func (self *CdrcConfig) Clone() *CdrcConfig {
+	clnCdrc := new(CdrcConfig)
+	clnCdrc.Enabled = self.Enabled
+	clnCdrc.Cdrs = self.Cdrs
+	clnCdrc.CdrFormat = self.CdrFormat
+	clnCdrc.FieldSeparator = self.FieldSeparator
+	clnCdrc.DataUsageMultiplyFactor = self.DataUsageMultiplyFactor
+	clnCdrc.RunDelay = self.RunDelay
+	clnCdrc.MaxOpenFiles = self.MaxOpenFiles
+	clnCdrc.CdrInDir = self.CdrInDir
+	clnCdrc.CdrOutDir = self.CdrOutDir
+	clnCdrc.CdrSourceId = self.CdrSourceId
+	clnCdrc.HeaderFields = make([]*CfgCdrField, len(self.HeaderFields))
+	clnCdrc.ContentFields = make([]*CfgCdrField, len(self.ContentFields))
+	clnCdrc.TrailerFields = make([]*CfgCdrField, len(self.TrailerFields))
+	for idx, fld := range self.HeaderFields {
+		clonedVal := *fld
+		clnCdrc.HeaderFields[idx] = &clonedVal
+	}
+	for idx, fld := range self.ContentFields {
+		clonedVal := *fld
+		clnCdrc.ContentFields[idx] = &clonedVal
+	}
+	for idx, fld := range self.TrailerFields {
+		clonedVal := *fld
+		clnCdrc.TrailerFields[idx] = &clonedVal
+	}
+	return clnCdrc
+}

config/cdrcconfig_test.go

@@ -0,0 +1,19 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config

config/cdreconfig.go

@@ -0,0 +1,126 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+// One instance of CdrExporter
+type CdreConfig struct {
+	CdrFormat                  string
+	FieldSeparator             rune
+	DataUsageMultiplyFactor    float64
+	SmsUsageMultiplyFactor     float64
+	GenericUsageMultiplyFactor float64
+	CostMultiplyFactor         float64
+	CostRoundingDecimals       int
+	CostShiftDigits            int
+	MaskDestId                 string
+	MaskLength                 int
+	ExportDir                  string
+	HeaderFields               []*CfgCdrField
+	ContentFields              []*CfgCdrField
+	TrailerFields              []*CfgCdrField
+}
+
+func (self *CdreConfig) loadFromJsonCfg(jsnCfg *CdreJsonCfg) error {
+	if jsnCfg == nil {
+		return nil
+	}
+	var err error
+	if jsnCfg.Cdr_format != nil {
+		self.CdrFormat = *jsnCfg.Cdr_format
+	}
+	if jsnCfg.Field_separator != nil && len(*jsnCfg.Field_separator) > 0 { // Make sure we got at least one character so we don't get panic here
+		sepStr := *jsnCfg.Field_separator
+		self.FieldSeparator = rune(sepStr[0])
+	}
+	if jsnCfg.Data_usage_multiply_factor != nil {
+		self.DataUsageMultiplyFactor = *jsnCfg.Data_usage_multiply_factor
+	}
+	if jsnCfg.Sms_usage_multiply_factor != nil {
+		self.SmsUsageMultiplyFactor = *jsnCfg.Sms_usage_multiply_factor
+	}
+	if jsnCfg.Generic_usage_multiply_factor != nil {
+		self.GenericUsageMultiplyFactor = *jsnCfg.Generic_usage_multiply_factor
+	}
+	if jsnCfg.Cost_multiply_factor != nil {
+		self.CostMultiplyFactor = *jsnCfg.Cost_multiply_factor
+	}
+	if jsnCfg.Cost_rounding_decimals != nil {
+		self.CostRoundingDecimals = *jsnCfg.Cost_rounding_decimals
+	}
+	if jsnCfg.Cost_shift_digits != nil {
+		self.CostShiftDigits = *jsnCfg.Cost_shift_digits
+	}
+	if jsnCfg.Mask_destination_id != nil {
+		self.MaskDestId = *jsnCfg.Mask_destination_id
+	}
+	if jsnCfg.Mask_length != nil {
+		self.MaskLength = *jsnCfg.Mask_length
+	}
+	if jsnCfg.Export_dir != nil {
+		self.ExportDir = *jsnCfg.Export_dir
+	}
+	if jsnCfg.Header_fields != nil {
+		if self.HeaderFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Header_fields); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Content_fields != nil {
+		if self.ContentFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Content_fields); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Trailer_fields != nil {
+		if self.TrailerFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Trailer_fields); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Clone itself into a new CdreConfig
+func (self *CdreConfig) Clone() *CdreConfig {
+	clnCdre := new(CdreConfig)
+	clnCdre.CdrFormat = self.CdrFormat
+	clnCdre.FieldSeparator = self.FieldSeparator
+	clnCdre.DataUsageMultiplyFactor = self.DataUsageMultiplyFactor
+	clnCdre.SmsUsageMultiplyFactor = self.SmsUsageMultiplyFactor
+	clnCdre.GenericUsageMultiplyFactor = self.GenericUsageMultiplyFactor
+	clnCdre.CostMultiplyFactor = self.CostMultiplyFactor
+	clnCdre.CostRoundingDecimals = self.CostRoundingDecimals
+	clnCdre.CostShiftDigits = self.CostShiftDigits
+	clnCdre.MaskDestId = self.MaskDestId
+	clnCdre.MaskLength = self.MaskLength
+	clnCdre.ExportDir = self.ExportDir
+	clnCdre.HeaderFields = make([]*CfgCdrField, len(self.HeaderFields))
+	for idx, fld := range self.HeaderFields {
+		clonedVal := *fld
+		clnCdre.HeaderFields[idx] = &clonedVal
+	}
+	clnCdre.ContentFields = make([]*CfgCdrField, len(self.ContentFields))
+	for idx, fld := range self.ContentFields {
+		clonedVal := *fld
+		clnCdre.ContentFields[idx] = &clonedVal
+	}
+	clnCdre.TrailerFields = make([]*CfgCdrField, len(self.TrailerFields))
+	for idx, fld := range self.TrailerFields {
+		clonedVal := *fld
+		clnCdre.TrailerFields[idx] = &clonedVal
+	}
+	return clnCdre
+}

config/cdreconfig_test.go

@@ -0,0 +1,97 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+package config
+
+import (
+	"github.com/cgrates/cgrates/utils"
+	"reflect"
+	"testing"
+)
+
+func TestCdreCfgClone(t *testing.T) {
+	cgrIdRsrs, _ := utils.ParseRSRFields("cgrid", utils.INFIELD_SEP)
+	runIdRsrs, _ := utils.ParseRSRFields("mediation_runid", utils.INFIELD_SEP)
+	emptyFields := []*CfgCdrField{}
+	initContentFlds := []*CfgCdrField{
+		&CfgCdrField{Tag: "CgrId",
+			Type:       "cdrfield",
+			CdrFieldId: "cgrid",
+			Value:      cgrIdRsrs},
+		&CfgCdrField{Tag: "RunId",
+			Type:       "cdrfield",
+			CdrFieldId: "mediation_runid",
+			Value:      runIdRsrs},
+	}
+	initCdreCfg := &CdreConfig{
+		CdrFormat:               "csv",
+		FieldSeparator:          rune(','),
+		DataUsageMultiplyFactor: 1.0,
+		CostMultiplyFactor:      1.0,
+		CostRoundingDecimals:    -1,
+		CostShiftDigits:         0,
+		MaskDestId:              "MASKED_DESTINATIONS",
+		MaskLength:              0,
+		ExportDir:               "/var/log/cgrates/cdre",
+		ContentFields:           initContentFlds,
+	}
+	eClnContentFlds := []*CfgCdrField{
+		&CfgCdrField{Tag: "CgrId",
+			Type:       "cdrfield",
+			CdrFieldId: "cgrid",
+			Value:      cgrIdRsrs},
+		&CfgCdrField{Tag: "RunId",
+			Type:       "cdrfield",
+			CdrFieldId: "mediation_runid",
+			Value:      runIdRsrs},
+	}
+	eClnCdreCfg := &CdreConfig{
+		CdrFormat:               "csv",
+		FieldSeparator:          rune(','),
+		DataUsageMultiplyFactor: 1.0,
+		CostMultiplyFactor:      1.0,
+		CostRoundingDecimals:    -1,
+		CostShiftDigits:         0,
+		MaskDestId:              "MASKED_DESTINATIONS",
+		MaskLength:              0,
+		ExportDir:               "/var/log/cgrates/cdre",
+		HeaderFields:            emptyFields,
+		ContentFields:           eClnContentFlds,
+		TrailerFields:           emptyFields,
+	}
+	clnCdreCfg := initCdreCfg.Clone()
+	if !reflect.DeepEqual(eClnCdreCfg, clnCdreCfg) {
+		t.Errorf("Cloned result: %+v", clnCdreCfg)
+	}
+	initCdreCfg.DataUsageMultiplyFactor = 1024.0
+	if !reflect.DeepEqual(eClnCdreCfg, clnCdreCfg) { // MOdifying a field after clone should not affect cloned instance
+		t.Errorf("Cloned result: %+v", clnCdreCfg)
+	}
+	initContentFlds[0].Tag = "Destination"
+	if !reflect.DeepEqual(eClnCdreCfg, clnCdreCfg) { // MOdifying a field after clone should not affect cloned instance
+		t.Errorf("Cloned result: %+v", clnCdreCfg)
+	}
+	clnCdreCfg.CostShiftDigits = 2
+	if initCdreCfg.CostShiftDigits != 0 {
+		t.Error("Unexpected CostShiftDigits: ", initCdreCfg.CostShiftDigits)
+	}
+	clnCdreCfg.ContentFields[0].CdrFieldId = "destination"
+	if initCdreCfg.ContentFields[0].CdrFieldId != "cgrid" {
+		t.Error("Unexpected change of CdrFieldId: ", initCdreCfg.ContentFields[0].CdrFieldId)
+	}
+
+}

config/cdrstatsconfig.go

@@ -0,0 +1,49 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"time"
+)
+
+type CdrStatsConfig struct {
+	Id                  string        // Config id, unique per config instance
+	QueueLength         int           // Number of items in the stats buffer
+	TimeWindow          time.Duration // Will only keep the CDRs who's call setup time is not older than time.Now()-TimeWindow
+	SaveInterval        time.Duration
+	Metrics             []string    // ASR, ACD, ACC
+	SetupInterval       []time.Time // 2 or less items (>= start interval,< stop_interval)
+	TORs                []string
+	CdrHosts            []string
+	CdrSources          []string
+	ReqTypes            []string
+	Directions          []string
+	Tenants             []string
+	Categories          []string
+	Accounts            []string
+	Subjects            []string
+	DestinationPrefixes []string
+	UsageInterval       []time.Duration // 2 or less items (>= Usage, <Usage)
+	Suppliers           []string
+	DisconnectCauses    []string
+	MediationRunIds     []string
+	RatedAccounts       []string
+	RatedSubjects       []string
+	CostInterval        []float64 // 2 or less items, (>=Cost, <Cost)
+}

config/cfg_data.json

@@ -0,0 +1,51 @@
+{
+
+// Real-time Charging System for Telecom & ISP environments
+// Copyright (C) ITsysCOM GmbH
+//
+// This file contains the default configuration hardcoded into CGRateS.
+// This is what you get when you load CGRateS with an empty configuration file.
+
+"general": {
+	"default_reqtype": "*pseudoprepaid",				// default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated>
+},
+
+"cdrs": {
+	"enabled": true,						// start the CDR Server service:  <true|false>
+},
+
+"rater": {
+	"enabled": true,						// enable Rater service: <true|false>
+},
+
+"cdrc": {
+	"CDRC-CSV1": {
+		"enabled": true,							// enable CDR client functionality
+		"cdr_in_dir": "/tmp/cgrates/cdrc1/in",		// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/tmp/cgrates/cdrc1/out",	// absolute path towards the directory where processed CDRs will be moved
+		"cdr_source_id": "csv1",					// free form field, tag identifying the source of the CDRs within CDRS database
+	},
+	"CDRC-CSV2": {
+		"enabled": true,							// enable CDR client functionality
+		"cdr_in_dir": "/tmp/cgrates/cdrc2/in",		// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/tmp/cgrates/cdrc2/out",	// absolute path towards the directory where processed CDRs will be moved
+		"data_usage_multiply_factor": 0.000976563,
+		"run_delay": 1,
+		"cdr_source_id": "csv2",					// free form field, tag identifying the source of the CDRs within CDRS database
+		"content_fields":[							// import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
+			{"cdr_field_id": "tor", "value": "~7:s/^(voice|data|sms|generic)$/*$1/"},
+			{"cdr_field_id": "answer_time", "value": "1"},
+			{"cdr_field_id": "usage", "value": "~9:s/^(\\d+)$/${1}s/"},
+		],
+	},
+},
+
+"sm_freeswitch": {
+	"enabled": true,				// starts SessionManager service: <true|false>
+	"connections":[					// instantiate connections to multiple FreeSWITCH servers
+		{"server": "1.2.3.4:8021", "password": "ClueCon", "reconnects": 5},
+		{"server": "2.3.4.5:8021", "password": "ClueCon", "reconnects": 5},
+	],
+},
+
+}

config/cfg_data2.json

@@ -0,0 +1,30 @@
+{
+
+"cdrc": {
+	"CDRC-CSV2": {
+		"enabled": true,							// enable CDR client functionality
+		"cdr_in_dir": "/tmp/cgrates/cdrc2/in",		// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/tmp/cgrates/cdrc2/out",	// absolute path towards the directory where processed CDRs will be moved
+		"data_usage_multiply_factor": 0.000976563,
+		"cdr_source_id": "csv2",					// free form field, tag identifying the source of the CDRs within CDRS database
+		"content_fields":[								// import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
+			{"cdr_field_id": "tor", "value": "~7:s/^(voice|data|sms|generic)$/*$1/"},
+			{"cdr_field_id": "answer_time", "value": "2"},
+		],
+	},
+	"CDRC-CSV3": {
+		"enabled": true,							// enable CDR client functionality
+		"cdr_in_dir": "/tmp/cgrates/cdrc3/in",		// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/tmp/cgrates/cdrc3/out",	// absolute path towards the directory where processed CDRs will be moved
+		"cdr_source_id": "csv3",					// free form field, tag identifying the source of the CDRs within CDRS database
+	},
+},
+
+"sm_freeswitch": {
+	"enabled": true,				// starts SessionManager service: <true|false>
+	"connections":[					// instantiate connections to multiple FreeSWITCH servers
+		{"server": "2.3.4.5:8021", "password": "ClueCon", "reconnects": 5},
+	],
+},
+
+}

config/cfgcdrfield.go

@@ -0,0 +1,92 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"github.com/cgrates/cgrates/utils"
+)
+
+func NewCfgCdrFieldFromCdrFieldJsonCfg(jsnCfgFld *CdrFieldJsonCfg) (*CfgCdrField, error) {
+	var err error
+	cfgFld := new(CfgCdrField)
+	if jsnCfgFld.Tag != nil {
+		cfgFld.Tag = *jsnCfgFld.Tag
+	}
+	if jsnCfgFld.Type != nil {
+		cfgFld.Type = *jsnCfgFld.Type
+	}
+	if jsnCfgFld.Cdr_field_id != nil {
+		cfgFld.CdrFieldId = *jsnCfgFld.Cdr_field_id
+	}
+	if jsnCfgFld.Metatag_id != nil {
+		cfgFld.MetatagId = *jsnCfgFld.Metatag_id
+	}
+	if jsnCfgFld.Value != nil {
+		if cfgFld.Value, err = utils.ParseRSRFields(*jsnCfgFld.Value, utils.INFIELD_SEP); err != nil {
+			return nil, err
+		}
+	}
+	if jsnCfgFld.Field_filter != nil {
+		if cfgFld.FieldFilter, err = utils.ParseRSRFields(*jsnCfgFld.Field_filter, utils.INFIELD_SEP); err != nil {
+			return nil, err
+		}
+	}
+	if jsnCfgFld.Width != nil {
+		cfgFld.Width = *jsnCfgFld.Width
+	}
+	if jsnCfgFld.Strip != nil {
+		cfgFld.Strip = *jsnCfgFld.Strip
+	}
+	if jsnCfgFld.Padding != nil {
+		cfgFld.Padding = *jsnCfgFld.Padding
+	}
+	if jsnCfgFld.Layout != nil {
+		cfgFld.Layout = *jsnCfgFld.Layout
+	}
+	if jsnCfgFld.Mandatory != nil {
+		cfgFld.Mandatory = *jsnCfgFld.Mandatory
+	}
+	return cfgFld, nil
+}
+
+type CfgCdrField struct {
+	Tag         string // Identifier for the administrator
+	Type        string // Type of field
+	CdrFieldId  string // StoredCdr field name
+	MetatagId   string
+	Value       utils.RSRFields
+	FieldFilter utils.RSRFields
+	Width       int
+	Strip       string
+	Padding     string
+	Layout      string
+	Mandatory   bool
+}
+
+func CfgCdrFieldsFromCdrFieldsJsonCfg(jsnCfgFldss []*CdrFieldJsonCfg) ([]*CfgCdrField, error) {
+	retFields := make([]*CfgCdrField, len(jsnCfgFldss))
+	for idx, jsnFld := range jsnCfgFldss {
+		if cfgFld, err := NewCfgCdrFieldFromCdrFieldJsonCfg(jsnFld); err != nil {
+			return nil, err
+		} else {
+			retFields[idx] = cfgFld
+		}
+	}
+	return retFields, nil
+}

config/cfgcdrfield_test.go

@@ -0,0 +1,19 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config

config/config_defaults.go

@@ -0,0 +1,270 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+const CGRATES_CFG_JSON = `
+{
+
+// Real-time Charging System for Telecom & ISP environments
+// Copyright (C) ITsysCOM GmbH
+//
+// This file contains the default configuration hardcoded into CGRateS.
+// This is what you get when you load CGRateS with an empty configuration file.
+
+"general": {
+	"http_skip_tls_verify": false,			// if enabled Http Client will accept any TLS certificate
+	"rounding_decimals": 10,				// system level precision for floats
+	"dbdata_encoding": "msgpack",			// encoding used to store object data in strings: <msgpack|json>
+	"tpexport_dir": "/var/log/cgrates/tpe",	// path towards export folder for offline Tariff Plans
+	"default_reqtype": "*rated",			// default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated>
+	"default_category": "call",				// default Type of Record to consider when missing from requests
+	"default_tenant": "cgrates.org",		// default Tenant to consider when missing from requests
+	"default_subject": "cgrates",			// default rating Subject to consider when missing from requests
+	"connect_attempts": 3,                  // initial server connect attempts
+	"reconnects": -1,                       // number of retries in case of connection lost
+},
+
+
+"listen": {
+	"rpc_json": "127.0.0.1:2012",			// RPC JSON listening address
+	"rpc_gob": "127.0.0.1:2013",			// RPC GOB listening address
+	"http": "127.0.0.1:2080",				// HTTP listening address
+},
+
+
+"tariffplan_db": {							// database used to store active tariff plan configuration
+	"db_type": "redis",						// tariffplan_db type: <redis>
+	"db_host": "127.0.0.1",					// tariffplan_db host address
+	"db_port": 6379, 						// port to reach the tariffplan_db
+	"db_name": "10", 						// tariffplan_db name to connect to
+	"db_user": "", 							// sername to use when connecting to tariffplan_db
+	"db_passwd": "", 						// password to use when connecting to tariffplan_db
+},
+
+
+"data_db": {								// database used to store runtime data (eg: accounts, cdr stats)
+	"db_type": "redis",						// data_db type: <redis>
+	"db_host": "127.0.0.1",					// data_db host address
+	"db_port": 6379, 						// data_db port to reach the database
+	"db_name": "11", 						// data_db database name to connect to
+	"db_user": "", 							// username to use when connecting to data_db
+	"db_passwd": "", 						// password to use when connecting to data_db
+},
+
+
+"stor_db": {								// database used to store offline tariff plans and CDRs
+	"db_type": "mysql",						// stor database type to use: <mysql|postgres>
+	"db_host": "127.0.0.1",					// the host to connect to
+	"db_port": 3306, 						// the port to reach the stordb
+	"db_name": "cgrates", 					// stor database name
+	"db_user": "cgrates", 					// username to use when connecting to stordb
+	"db_passwd": "CGRateS.org", 			// password to use when connecting to stordb
+	"max_open_conns": 100,					// maximum database connections opened
+	"max_idle_conns": 10,					// maximum database connections idle
+},
+
+
+"balancer": {
+	"enabled": false, 						// start Balancer service: <true|false>
+},
+
+
+"rater": {
+	"enabled": false,						// enable Rater service: <true|false>
+	"balancer": "",							// register to balancer as worker: <""|internal|x.y.z.y:1234>
+	"cdrstats": "",							// address where to reach the cdrstats service, empty to disable stats functionality: <""|internal|x.y.z.y:1234>
+	"historys": "",							// address where to reach the history service, empty to disable history functionality: <""|internal|x.y.z.y:1234>
+	"pubsubs": "",							// address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234>
+	"users": "",							// address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234>
+},
+
+
+"scheduler": {
+	"enabled": false,						// start Scheduler service: <true|false>
+},
+
+
+"cdrs": {
+	"enabled": false,						// start the CDR Server service:  <true|false>
+	"extra_fields": [],						// extra fields to store in CDRs for non-generic CDRs
+	"store_cdrs": true,						// store cdrs in storDb
+	"rater": "internal",					// address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234>
+	"cdrstats": "",							// address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234>
+	"reconnects": 5,						// number of reconnect attempts to rater or cdrs
+	"cdr_replication":[],					// replicate the raw CDR to a number of servers
+},
+
+
+"cdrstats": {
+	"enabled": false,						// starts the cdrstats service: <true|false>
+	"save_interval": "1m",					// interval to save changed stats into dataDb storage
+},
+
+
+"cdre": {
+	"*default": {
+		"cdr_format": "csv",							// exported CDRs format <csv>
+		"field_separator": ",",
+		"data_usage_multiply_factor": 1,				// multiply data usage before export (eg: convert from KBytes to Bytes)
+		"sms_usage_multiply_factor": 1,					// multiply data usage before export (eg: convert from SMS unit to call duration in some billing systems)
+        "generic_usage_multiply_factor": 1,					// multiply data usage before export (eg: convert from GENERIC unit to call duration in some billing systems)
+		"cost_multiply_factor": 1,						// multiply cost before export, eg: add VAT
+		"cost_rounding_decimals": -1,					// rounding decimals for Cost values. -1 to disable rounding
+		"cost_shift_digits": 0,							// shift digits in the cost on export (eg: convert from EUR to cents)
+		"mask_destination_id": "MASKED_DESTINATIONS",	// destination id containing called addresses to be masked on export
+		"mask_length": 0,								// length of the destination suffix to be masked
+		"export_dir": "/var/log/cgrates/cdre",			// path where the exported CDRs will be placed
+		"header_fields": [],							// template of the exported header fields
+		"content_fields": [								// template of the exported content fields
+			{"tag": "CgrId", "cdr_field_id": "cgrid", "type": "cdrfield", "value": "cgrid"},
+			{"tag":"RunId", "cdr_field_id": "mediation_runid", "type": "cdrfield", "value": "mediation_runid"},
+			{"tag":"Tor", "cdr_field_id": "tor", "type": "cdrfield", "value": "tor"},
+			{"tag":"AccId", "cdr_field_id": "accid", "type": "cdrfield", "value": "accid"},
+			{"tag":"ReqType", "cdr_field_id": "reqtype", "type": "cdrfield", "value": "reqtype"},
+			{"tag":"Direction", "cdr_field_id": "direction", "type": "cdrfield", "value": "direction"},
+			{"tag":"Tenant", "cdr_field_id": "tenant", "type": "cdrfield", "value": "tenant"},
+			{"tag":"Category", "cdr_field_id": "category", "type": "cdrfield", "value": "category"},
+			{"tag":"Account", "cdr_field_id": "account", "type": "cdrfield", "value": "account"},
+			{"tag":"Subject", "cdr_field_id": "subject", "type": "cdrfield", "value": "subject"},
+			{"tag":"Destination", "cdr_field_id": "destination", "type": "cdrfield", "value": "destination"},
+			{"tag":"SetupTime", "cdr_field_id": "setup_time", "type": "cdrfield", "value": "setup_time", "layout": "2006-01-02T15:04:05Z07:00"},
+			{"tag":"AnswerTime", "cdr_field_id": "answer_time", "type": "cdrfield", "value": "answer_time", "layout": "2006-01-02T15:04:05Z07:00"},
+			{"tag":"Usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "usage"},
+			{"tag":"Cost", "cdr_field_id": "cost", "type": "cdrfield", "value": "cost"},
+		],
+		"trailer_fields": [],							// template of the exported trailer fields
+	}
+},
+
+
+"cdrc": {
+	"*default": {
+		"enabled": false,							// enable CDR client functionality
+		"dry_run": false,							// do not send the CDRs to CDRS, just parse them
+		"cdrs": "internal",							// address where to reach CDR server. <internal|x.y.z.y:1234>
+		"cdr_format": "csv",						// CDR file format <csv|freeswitch_csv|fwv|opensips_flatstore>
+		"field_separator": ",",						// separator used in case of csv files
+		"run_delay": 0,								// sleep interval in seconds between consecutive runs, 0 to use automation via inotify
+		"max_open_files": 1024,						// maximum simultaneous files to process, 0 for unlimited
+		"data_usage_multiply_factor": 1024,			// conversion factor for data usage
+		"cdr_in_dir": "/var/log/cgrates/cdrc/in",	// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/var/log/cgrates/cdrc/out",	// absolute path towards the directory where processed CDRs will be moved
+		"failed_calls_prefix": "missed_calls",		// used in case of flatstore CDRs to avoid searching for BYE records
+		"cdr_source_id": "freeswitch_csv",			// free form field, tag identifying the source of the CDRs within CDRS database
+		"cdr_filter": "",							// filter CDR records to import
+		"partial_record_cache": "10s",				// duration to cache partial records when not pairing
+		"header_fields": [],						// template of the import header fields
+		"content_fields":[							// import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
+			{"tag": "tor", "cdr_field_id": "tor", "type": "cdrfield", "value": "2", "mandatory": true},
+			{"tag": "accid", "cdr_field_id": "accid", "type": "cdrfield", "value": "3", "mandatory": true},
+			{"tag": "reqtype", "cdr_field_id": "reqtype", "type": "cdrfield", "value": "4", "mandatory": true},
+			{"tag": "direction", "cdr_field_id": "direction", "type": "cdrfield", "value": "5", "mandatory": true},
+			{"tag": "tenant", "cdr_field_id": "tenant", "type": "cdrfield", "value": "6", "mandatory": true},
+			{"tag": "category", "cdr_field_id": "category", "type": "cdrfield", "value": "7", "mandatory": true},
+			{"tag": "account", "cdr_field_id": "account", "type": "cdrfield", "value": "8", "mandatory": true},
+			{"tag": "subject", "cdr_field_id": "subject", "type": "cdrfield", "value": "9", "mandatory": true},
+			{"tag": "destination", "cdr_field_id": "destination", "type": "cdrfield", "value": "10", "mandatory": true},
+			{"tag": "setup_time", "cdr_field_id": "setup_time", "type": "cdrfield", "value": "11", "mandatory": true},
+			{"tag": "answer_time", "cdr_field_id": "answer_time", "type": "cdrfield", "value": "12", "mandatory": true},
+			{"tag": "usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "13", "mandatory": true},
+		],
+		"trailer_fields": [],							// template of the import trailer fields
+	}
+},
+
+
+"sm_freeswitch": {
+	"enabled": false,				// starts SessionManager service: <true|false>
+	"rater": "internal",			// address where to reach the Rater <""|internal|127.0.0.1:2013>
+	"cdrs": "internal",				// address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234>
+	"reconnects": 5,				// number of reconnect attempts to rater or cdrs
+	"create_cdr": false,			// create CDR out of events and sends them to CDRS component
+	"cdr_extra_fields": [],			// extra fields to store in CDRs when creating them
+	"debit_interval": "10s",		// interval to perform debits on.
+	"min_call_duration": "0s",		// only authorize calls with allowed duration higher than this
+	"max_call_duration": "3h",		// maximum call duration a prepaid call can last
+	"min_dur_low_balance": "5s",	// threshold which will trigger low balance warnings for prepaid calls (needs to be lower than debit_interval)
+	"low_balance_ann_file": "",		// file to be played when low balance is reached for prepaid calls
+	"empty_balance_context": "",	// if defined, prepaid calls will be transfered to this context on empty balance
+	"empty_balance_ann_file": "",	// file to be played before disconnecting prepaid calls on empty balance (applies only if no context defined)
+	"subscribe_park": true,			// subscribe via fsock to receive park events
+	"channel_sync_interval": "5m",	// sync channels with freeswitch regularly
+	"connections":[					// instantiate connections to multiple FreeSWITCH servers
+		{"server": "127.0.0.1:8021", "password": "ClueCon", "reconnects": 5}
+	],
+},
+
+
+"sm_kamailio": {
+	"enabled": false,				// starts SessionManager service: <true|false>
+	"rater": "internal",			// address where to reach the Rater <""|internal|127.0.0.1:2013>
+	"cdrs": "internal",				// address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234>
+	"reconnects": 5,				// number of reconnect attempts to rater or cdrs
+	"create_cdr": false,			// create CDR out of events and sends them to CDRS component
+	"debit_interval": "10s",		// interval to perform debits on.
+	"min_call_duration": "0s",		// only authorize calls with allowed duration higher than this
+	"max_call_duration": "3h",		// maximum call duration a prepaid call can last
+	"connections":[					// instantiate connections to multiple Kamailio servers
+		{"evapi_addr": "127.0.0.1:8448", "reconnects": 5}
+	],
+},
+
+
+"sm_opensips": {
+	"enabled": false,					// starts SessionManager service: <true|false>
+	"listen_udp": "127.0.0.1:2020",		// address where to listen for datagram events coming from OpenSIPS
+	"rater": "internal",				// address where to reach the Rater <""|internal|127.0.0.1:2013>
+	"cdrs": "internal",					// address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234>
+	"reconnects": 5,					// number of reconnects if connection is lost
+	"create_cdr": false,				// create CDR out of events and sends them to CDRS component
+	"debit_interval": "10s",			// interval to perform debits on.
+	"min_call_duration": "0s",			// only authorize calls with allowed duration higher than this
+	"max_call_duration": "3h",			// maximum call duration a prepaid call can last
+	"events_subscribe_interval": "60s",	// automatic events subscription to OpenSIPS, 0 to disable it
+	"mi_addr": "127.0.0.1:8020",		// address where to reach OpenSIPS MI to send session disconnects
+},
+
+
+"historys": {
+	"enabled": false,							// starts History service: <true|false>.
+	"history_dir": "/var/log/cgrates/history",	// location on disk where to store history files.
+	"save_interval": "1s",						// interval to save changed cache into .git archive
+},
+
+
+"pubsubs": {
+	"enabled": false,							// starts PubSub service: <true|false>.
+},
+
+
+"users": {
+	"enabled": false,							// starts User service: <true|false>.
+	"indexes": [],                  			// user profile field indexes
+},
+
+
+"mailer": {
+	"server": "localhost",								// the server to use when sending emails out
+	"auth_user": "cgrates",								// authenticate to email server using this user
+	"auth_passwd": "CGRateS.org",						// authenticate to email server with this password
+	"from_address": "cgr-mailer@localhost.localdomain"	// from address used when sending emails out
+},
+
+
+}`

config/config_json.go

@@ -0,0 +1,281 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+
+	"github.com/DisposaBoy/JsonConfigReader"
+)
+
+const (
+	GENERAL_JSN    = "general"
+	LISTEN_JSN     = "listen"
+	TPDB_JSN       = "tariffplan_db"
+	DATADB_JSN     = "data_db"
+	STORDB_JSN     = "stor_db"
+	BALANCER_JSN   = "balancer"
+	RATER_JSN      = "rater"
+	SCHEDULER_JSN  = "scheduler"
+	CDRS_JSN       = "cdrs"
+	MEDIATOR_JSN   = "mediator"
+	CDRSTATS_JSN   = "cdrstats"
+	CDRE_JSN       = "cdre"
+	CDRC_JSN       = "cdrc"
+	SMFS_JSN       = "sm_freeswitch"
+	SMKAM_JSN      = "sm_kamailio"
+	SMOSIPS_JSN    = "sm_opensips"
+	SM_JSN         = "session_manager"
+	FS_JSN         = "freeswitch"
+	KAMAILIO_JSN   = "kamailio"
+	OSIPS_JSN      = "opensips"
+	HISTSERV_JSN   = "historys"
+	PUBSUBSERV_JSN = "pubsubs"
+	USERSERV_JSN   = "users"
+	MAILER_JSN     = "mailer"
+)
+
+// Loads the json config out of io.Reader, eg other sources than file, maybe over http
+func NewCgrJsonCfgFromReader(r io.Reader) (*CgrJsonCfg, error) {
+	var cgrJsonCfg CgrJsonCfg
+	jr := JsonConfigReader.New(r)
+	if err := json.NewDecoder(jr).Decode(&cgrJsonCfg); err != nil {
+		return nil, err
+	}
+	return &cgrJsonCfg, nil
+}
+
+// Loads the config out of file
+func NewCgrJsonCfgFromFile(fpath string) (*CgrJsonCfg, error) {
+	cfgFile, err := os.Open(fpath)
+	if err != nil {
+		return nil, err
+	}
+	defer cfgFile.Close()
+	return NewCgrJsonCfgFromReader(cfgFile)
+}
+
+// Main object holding the loaded config as section raw messages
+type CgrJsonCfg map[string]*json.RawMessage
+
+func (self CgrJsonCfg) GeneralJsonCfg() (*GeneralJsonCfg, error) {
+	rawCfg, hasKey := self[GENERAL_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(GeneralJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) ListenJsonCfg() (*ListenJsonCfg, error) {
+	rawCfg, hasKey := self[LISTEN_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(ListenJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) DbJsonCfg(section string) (*DbJsonCfg, error) {
+	rawCfg, hasKey := self[section]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(DbJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) BalancerJsonCfg() (*BalancerJsonCfg, error) {
+	rawCfg, hasKey := self[BALANCER_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(BalancerJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) RaterJsonCfg() (*RaterJsonCfg, error) {
+	rawCfg, hasKey := self[RATER_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(RaterJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) SchedulerJsonCfg() (*SchedulerJsonCfg, error) {
+	rawCfg, hasKey := self[SCHEDULER_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(SchedulerJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) CdrsJsonCfg() (*CdrsJsonCfg, error) {
+	rawCfg, hasKey := self[CDRS_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(CdrsJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) CdrStatsJsonCfg() (*CdrStatsJsonCfg, error) {
+	rawCfg, hasKey := self[CDRSTATS_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(CdrStatsJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) CdreJsonCfgs() (map[string]*CdreJsonCfg, error) {
+	rawCfg, hasKey := self[CDRE_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := make(map[string]*CdreJsonCfg)
+	if err := json.Unmarshal(*rawCfg, &cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) CdrcJsonCfg() (map[string]*CdrcJsonCfg, error) {
+	rawCfg, hasKey := self[CDRC_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := make(map[string]*CdrcJsonCfg)
+	if err := json.Unmarshal(*rawCfg, &cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) SmFsJsonCfg() (*SmFsJsonCfg, error) {
+	rawCfg, hasKey := self[SMFS_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(SmFsJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) SmKamJsonCfg() (*SmKamJsonCfg, error) {
+	rawCfg, hasKey := self[SMKAM_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(SmKamJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) SmOsipsJsonCfg() (*SmOsipsJsonCfg, error) {
+	rawCfg, hasKey := self[SMOSIPS_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(SmOsipsJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) HistServJsonCfg() (*HistServJsonCfg, error) {
+	rawCfg, hasKey := self[HISTSERV_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(HistServJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) PubSubServJsonCfg() (*PubSubServJsonCfg, error) {
+	rawCfg, hasKey := self[PUBSUBSERV_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(PubSubServJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) UserServJsonCfg() (*UserServJsonCfg, error) {
+	rawCfg, hasKey := self[USERSERV_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(UserServJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) MailerJsonCfg() (*MailerJsonCfg, error) {
+	rawCfg, hasKey := self[MAILER_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(MailerJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}

config/config_json_test.go

@@ -0,0 +1,503 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+var dfCgrJsonCfg *CgrJsonCfg
+
+// Loads up the default configuration and  tests it's sections one by one
+func TestDfNewdfCgrJsonCfgFromReader(t *testing.T) {
+	var err error
+	if dfCgrJsonCfg, err = NewCgrJsonCfgFromReader(strings.NewReader(CGRATES_CFG_JSON)); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestDfGeneralJsonCfg(t *testing.T) {
+	eCfg := &GeneralJsonCfg{
+		Http_skip_tls_verify: utils.BoolPointer(false),
+		Rounding_decimals:    utils.IntPointer(10),
+		Dbdata_encoding:      utils.StringPointer("msgpack"),
+		Tpexport_dir:         utils.StringPointer("/var/log/cgrates/tpe"),
+		Default_reqtype:      utils.StringPointer(utils.META_RATED),
+		Default_category:     utils.StringPointer("call"),
+		Default_tenant:       utils.StringPointer("cgrates.org"),
+		Default_subject:      utils.StringPointer("cgrates"),
+		Connect_attempts:     utils.IntPointer(3),
+		Reconnects:           utils.IntPointer(-1)}
+	if gCfg, err := dfCgrJsonCfg.GeneralJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, gCfg) {
+		t.Error("Received: ", gCfg)
+	}
+}
+
+func TestDfListenJsonCfg(t *testing.T) {
+	eCfg := &ListenJsonCfg{
+		Rpc_json: utils.StringPointer("127.0.0.1:2012"),
+		Rpc_gob:  utils.StringPointer("127.0.0.1:2013"),
+		Http:     utils.StringPointer("127.0.0.1:2080")}
+	if cfg, err := dfCgrJsonCfg.ListenJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfDbJsonCfg(t *testing.T) {
+	eCfg := &DbJsonCfg{
+		Db_type:   utils.StringPointer("redis"),
+		Db_host:   utils.StringPointer("127.0.0.1"),
+		Db_port:   utils.IntPointer(6379),
+		Db_name:   utils.StringPointer("10"),
+		Db_user:   utils.StringPointer(""),
+		Db_passwd: utils.StringPointer(""),
+	}
+	if cfg, err := dfCgrJsonCfg.DbJsonCfg(TPDB_JSN); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+	eCfg = &DbJsonCfg{
+		Db_type:   utils.StringPointer("redis"),
+		Db_host:   utils.StringPointer("127.0.0.1"),
+		Db_port:   utils.IntPointer(6379),
+		Db_name:   utils.StringPointer("11"),
+		Db_user:   utils.StringPointer(""),
+		Db_passwd: utils.StringPointer(""),
+	}
+	if cfg, err := dfCgrJsonCfg.DbJsonCfg(DATADB_JSN); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+	eCfg = &DbJsonCfg{
+		Db_type:        utils.StringPointer("mysql"),
+		Db_host:        utils.StringPointer("127.0.0.1"),
+		Db_port:        utils.IntPointer(3306),
+		Db_name:        utils.StringPointer("cgrates"),
+		Db_user:        utils.StringPointer("cgrates"),
+		Db_passwd:      utils.StringPointer("CGRateS.org"),
+		Max_open_conns: utils.IntPointer(100),
+		Max_idle_conns: utils.IntPointer(10),
+	}
+	if cfg, err := dfCgrJsonCfg.DbJsonCfg(STORDB_JSN); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfBalancerJsonCfg(t *testing.T) {
+	eCfg := &BalancerJsonCfg{Enabled: utils.BoolPointer(false)}
+	if cfg, err := dfCgrJsonCfg.BalancerJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfRaterJsonCfg(t *testing.T) {
+	eCfg := &RaterJsonCfg{Enabled: utils.BoolPointer(false), Balancer: utils.StringPointer(""), Cdrstats: utils.StringPointer(""),
+		Historys: utils.StringPointer(""), Pubsubs: utils.StringPointer(""), Users: utils.StringPointer("")}
+	if cfg, err := dfCgrJsonCfg.RaterJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfSchedulerJsonCfg(t *testing.T) {
+	eCfg := &SchedulerJsonCfg{Enabled: utils.BoolPointer(false)}
+	if cfg, err := dfCgrJsonCfg.SchedulerJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfCdrsJsonCfg(t *testing.T) {
+	eCfg := &CdrsJsonCfg{
+		Enabled:         utils.BoolPointer(false),
+		Extra_fields:    utils.StringSlicePointer([]string{}),
+		Store_cdrs:      utils.BoolPointer(true),
+		Rater:           utils.StringPointer("internal"),
+		Cdrstats:        utils.StringPointer(""),
+		Reconnects:      utils.IntPointer(5),
+		Cdr_replication: &[]*CdrReplicationJsonCfg{},
+	}
+	if cfg, err := dfCgrJsonCfg.CdrsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", *cfg)
+	}
+}
+
+func TestDfCdrStatsJsonCfg(t *testing.T) {
+	eCfg := &CdrStatsJsonCfg{
+		Enabled:       utils.BoolPointer(false),
+		Save_Interval: utils.StringPointer("1m"),
+	}
+	if cfg, err := dfCgrJsonCfg.CdrStatsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", *cfg)
+	}
+}
+
+func TestDfCdreJsonCfgs(t *testing.T) {
+	eFields := []*CdrFieldJsonCfg{}
+	eContentFlds := []*CdrFieldJsonCfg{
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("CgrId"),
+			Cdr_field_id: utils.StringPointer("cgrid"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("cgrid")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("RunId"),
+			Cdr_field_id: utils.StringPointer("mediation_runid"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("mediation_runid")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Tor"),
+			Cdr_field_id: utils.StringPointer("tor"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("tor")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("AccId"),
+			Cdr_field_id: utils.StringPointer("accid"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("accid")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("ReqType"),
+			Cdr_field_id: utils.StringPointer("reqtype"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("reqtype")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Direction"),
+			Cdr_field_id: utils.StringPointer("direction"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("direction")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Tenant"),
+			Cdr_field_id: utils.StringPointer("tenant"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("tenant")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Category"),
+			Cdr_field_id: utils.StringPointer("category"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("category")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Account"),
+			Cdr_field_id: utils.StringPointer("account"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("account")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Subject"),
+			Cdr_field_id: utils.StringPointer("subject"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("subject")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Destination"),
+			Cdr_field_id: utils.StringPointer("destination"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("destination")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("SetupTime"),
+			Cdr_field_id: utils.StringPointer("setup_time"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("setup_time"),
+			Layout:       utils.StringPointer("2006-01-02T15:04:05Z07:00")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("AnswerTime"),
+			Cdr_field_id: utils.StringPointer("answer_time"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("answer_time"),
+			Layout:       utils.StringPointer("2006-01-02T15:04:05Z07:00")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Usage"),
+			Cdr_field_id: utils.StringPointer("usage"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("usage")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Cost"),
+			Cdr_field_id: utils.StringPointer("cost"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("cost")},
+	}
+	eCfg := map[string]*CdreJsonCfg{
+		utils.META_DEFAULT: &CdreJsonCfg{
+			Cdr_format:                    utils.StringPointer("csv"),
+			Field_separator:               utils.StringPointer(","),
+			Data_usage_multiply_factor:    utils.Float64Pointer(1.0),
+			Sms_usage_multiply_factor:     utils.Float64Pointer(1.0),
+			Generic_usage_multiply_factor: utils.Float64Pointer(1.0),
+			Cost_multiply_factor:          utils.Float64Pointer(1.0),
+			Cost_rounding_decimals:        utils.IntPointer(-1),
+			Cost_shift_digits:             utils.IntPointer(0),
+			Mask_destination_id:           utils.StringPointer("MASKED_DESTINATIONS"),
+			Mask_length:                   utils.IntPointer(0),
+			Export_dir:                    utils.StringPointer("/var/log/cgrates/cdre"),
+			Header_fields:                 &eFields,
+			Content_fields:                &eContentFlds,
+			Trailer_fields:                &eFields,
+		},
+	}
+	if cfg, err := dfCgrJsonCfg.CdreJsonCfgs(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfCdrcJsonCfg(t *testing.T) {
+	eFields := []*CdrFieldJsonCfg{}
+	cdrFields := []*CdrFieldJsonCfg{
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("tor"), Cdr_field_id: utils.StringPointer("tor"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("2"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("accid"), Cdr_field_id: utils.StringPointer("accid"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("3"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("reqtype"), Cdr_field_id: utils.StringPointer("reqtype"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("4"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("direction"), Cdr_field_id: utils.StringPointer("direction"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("5"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("tenant"), Cdr_field_id: utils.StringPointer("tenant"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("6"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("category"), Cdr_field_id: utils.StringPointer("category"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("7"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("account"), Cdr_field_id: utils.StringPointer("account"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("8"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("subject"), Cdr_field_id: utils.StringPointer("subject"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("9"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("destination"), Cdr_field_id: utils.StringPointer("destination"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("10"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("setup_time"), Cdr_field_id: utils.StringPointer("setup_time"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("11"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("answer_time"), Cdr_field_id: utils.StringPointer("answer_time"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("12"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("usage"), Cdr_field_id: utils.StringPointer("usage"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("13"), Mandatory: utils.BoolPointer(true)},
+	}
+	eCfg := map[string]*CdrcJsonCfg{
+		"*default": &CdrcJsonCfg{
+			Enabled:                    utils.BoolPointer(false),
+			Dry_run:                    utils.BoolPointer(false),
+			Cdrs:                       utils.StringPointer("internal"),
+			Cdr_format:                 utils.StringPointer("csv"),
+			Field_separator:            utils.StringPointer(","),
+			Run_delay:                  utils.IntPointer(0),
+			Max_open_files:             utils.IntPointer(1024),
+			Data_usage_multiply_factor: utils.Float64Pointer(1024.0),
+			Cdr_in_dir:                 utils.StringPointer("/var/log/cgrates/cdrc/in"),
+			Cdr_out_dir:                utils.StringPointer("/var/log/cgrates/cdrc/out"),
+			Failed_calls_prefix:        utils.StringPointer("missed_calls"),
+			Cdr_source_id:              utils.StringPointer("freeswitch_csv"),
+			Cdr_filter:                 utils.StringPointer(""),
+			Partial_record_cache:       utils.StringPointer("10s"),
+			Header_fields:              &eFields,
+			Content_fields:             &cdrFields,
+			Trailer_fields:             &eFields,
+		},
+	}
+	if cfg, err := dfCgrJsonCfg.CdrcJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg["*default"])
+	}
+}
+
+func TestSmFsJsonCfg(t *testing.T) {
+	eCfg := &SmFsJsonCfg{
+		Enabled:                utils.BoolPointer(false),
+		Rater:                  utils.StringPointer("internal"),
+		Cdrs:                   utils.StringPointer("internal"),
+		Reconnects:             utils.IntPointer(5),
+		Create_cdr:             utils.BoolPointer(false),
+		Cdr_extra_fields:       utils.StringSlicePointer([]string{}),
+		Debit_interval:         utils.StringPointer("10s"),
+		Min_call_duration:      utils.StringPointer("0s"),
+		Max_call_duration:      utils.StringPointer("3h"),
+		Min_dur_low_balance:    utils.StringPointer("5s"),
+		Low_balance_ann_file:   utils.StringPointer(""),
+		Empty_balance_context:  utils.StringPointer(""),
+		Empty_balance_ann_file: utils.StringPointer(""),
+		Subscribe_park:         utils.BoolPointer(true),
+		Channel_sync_interval:  utils.StringPointer("5m"),
+		Connections: &[]*FsConnJsonCfg{
+			&FsConnJsonCfg{
+				Server:     utils.StringPointer("127.0.0.1:8021"),
+				Password:   utils.StringPointer("ClueCon"),
+				Reconnects: utils.IntPointer(5),
+			}},
+	}
+	if cfg, err := dfCgrJsonCfg.SmFsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestSmKamJsonCfg(t *testing.T) {
+	eCfg := &SmKamJsonCfg{
+		Enabled:           utils.BoolPointer(false),
+		Rater:             utils.StringPointer("internal"),
+		Cdrs:              utils.StringPointer("internal"),
+		Reconnects:        utils.IntPointer(5),
+		Create_cdr:        utils.BoolPointer(false),
+		Debit_interval:    utils.StringPointer("10s"),
+		Min_call_duration: utils.StringPointer("0s"),
+		Max_call_duration: utils.StringPointer("3h"),
+		Connections: &[]*KamConnJsonCfg{
+			&KamConnJsonCfg{
+				Evapi_addr: utils.StringPointer("127.0.0.1:8448"),
+				Reconnects: utils.IntPointer(5),
+			},
+		},
+	}
+	if cfg, err := dfCgrJsonCfg.SmKamJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestSmOsipsJsonCfg(t *testing.T) {
+	eCfg := &SmOsipsJsonCfg{
+		Enabled:                   utils.BoolPointer(false),
+		Listen_udp:                utils.StringPointer("127.0.0.1:2020"),
+		Rater:                     utils.StringPointer("internal"),
+		Cdrs:                      utils.StringPointer("internal"),
+		Reconnects:                utils.IntPointer(5),
+		Create_cdr:                utils.BoolPointer(false),
+		Debit_interval:            utils.StringPointer("10s"),
+		Min_call_duration:         utils.StringPointer("0s"),
+		Max_call_duration:         utils.StringPointer("3h"),
+		Events_subscribe_interval: utils.StringPointer("60s"),
+		Mi_addr:                   utils.StringPointer("127.0.0.1:8020"),
+	}
+	if cfg, err := dfCgrJsonCfg.SmOsipsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfHistServJsonCfg(t *testing.T) {
+	eCfg := &HistServJsonCfg{
+		Enabled:       utils.BoolPointer(false),
+		History_dir:   utils.StringPointer("/var/log/cgrates/history"),
+		Save_interval: utils.StringPointer("1s"),
+	}
+	if cfg, err := dfCgrJsonCfg.HistServJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfPubSubServJsonCfg(t *testing.T) {
+	eCfg := &PubSubServJsonCfg{
+		Enabled: utils.BoolPointer(false),
+	}
+	if cfg, err := dfCgrJsonCfg.PubSubServJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfUserServJsonCfg(t *testing.T) {
+	eCfg := &UserServJsonCfg{
+		Enabled: utils.BoolPointer(false),
+		Indexes: utils.StringSlicePointer([]string{}),
+	}
+	if cfg, err := dfCgrJsonCfg.UserServJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfMailerJsonCfg(t *testing.T) {
+
+	eCfg := &MailerJsonCfg{
+		Server:       utils.StringPointer("localhost"),
+		Auth_user:    utils.StringPointer("cgrates"),
+		Auth_passwd:  utils.StringPointer("CGRateS.org"),
+		From_address: utils.StringPointer("cgr-mailer@localhost.localdomain"),
+	}
+	if cfg, err := dfCgrJsonCfg.MailerJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestNewCgrJsonCfgFromFile(t *testing.T) {
+	cgrJsonCfg, err := NewCgrJsonCfgFromFile("cfg_data.json")
+	if err != nil {
+		t.Error(err)
+	}
+	eCfg := &GeneralJsonCfg{Default_reqtype: utils.StringPointer(utils.META_PSEUDOPREPAID)}
+	if gCfg, err := cgrJsonCfg.GeneralJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, gCfg) {
+		t.Error("Received: ", gCfg)
+	}
+	cdrFields := []*CdrFieldJsonCfg{
+		&CdrFieldJsonCfg{Cdr_field_id: utils.StringPointer("tor"), Value: utils.StringPointer("~7:s/^(voice|data|sms|generic)$/*$1/")},
+		&CdrFieldJsonCfg{Cdr_field_id: utils.StringPointer("answer_time"), Value: utils.StringPointer("1")},
+		&CdrFieldJsonCfg{Cdr_field_id: utils.StringPointer("usage"), Value: utils.StringPointer(`~9:s/^(\d+)$/${1}s/`)},
+	}
+	eCfgCdrc := map[string]*CdrcJsonCfg{
+		"CDRC-CSV1": &CdrcJsonCfg{
+			Enabled:       utils.BoolPointer(true),
+			Cdr_in_dir:    utils.StringPointer("/tmp/cgrates/cdrc1/in"),
+			Cdr_out_dir:   utils.StringPointer("/tmp/cgrates/cdrc1/out"),
+			Cdr_source_id: utils.StringPointer("csv1"),
+		},
+		"CDRC-CSV2": &CdrcJsonCfg{
+			Enabled:                    utils.BoolPointer(true),
+			Data_usage_multiply_factor: utils.Float64Pointer(0.000976563),
+			Run_delay:                  utils.IntPointer(1),
+			Cdr_in_dir:                 utils.StringPointer("/tmp/cgrates/cdrc2/in"),
+			Cdr_out_dir:                utils.StringPointer("/tmp/cgrates/cdrc2/out"),
+			Cdr_source_id:              utils.StringPointer("csv2"),
+			Content_fields:             &cdrFields,
+		},
+	}
+	if cfg, err := cgrJsonCfg.CdrcJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfgCdrc, cfg) {
+		t.Error("Received: ", cfg["CDRC-CSV2"])
+	}
+	eCfgSmFs := &SmFsJsonCfg{
+		Enabled: utils.BoolPointer(true),
+		Connections: &[]*FsConnJsonCfg{
+			&FsConnJsonCfg{
+				Server:     utils.StringPointer("1.2.3.4:8021"),
+				Password:   utils.StringPointer("ClueCon"),
+				Reconnects: utils.IntPointer(5),
+			},
+			&FsConnJsonCfg{
+				Server:     utils.StringPointer("2.3.4.5:8021"),
+				Password:   utils.StringPointer("ClueCon"),
+				Reconnects: utils.IntPointer(5),
+			},
+		},
+	}
+	if smFsCfg, err := cgrJsonCfg.SmFsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfgSmFs, smFsCfg) {
+		t.Error("Received: ", smFsCfg)
+	}
+}

config/config_local_test.go

@@ -1,47 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package config
-
-import (
-	"flag"
-	"path"
-	"testing"
-)
-
-var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args
-var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here")
-
-func TestLoadXmlCfg(t *testing.T) {
-	if !*testLocal {
-		return
-	}
-	cfgPath := path.Join(*dataDir, "conf", "samples", "config_local_test.cfg")
-	cfg, err := NewCGRConfig(&cfgPath)
-	if err != nil {
-		t.Error(err)
-	}
-	if cfg.XmlCfgDocument == nil {
-		t.Error("Did not load the XML Config Document")
-	}
-	if cdreFWCfg, err := cfg.XmlCfgDocument.GetCdreFWCfg("CDREFW-A"); err != nil {
-		t.Error(err)
-	} else if cdreFWCfg == nil {
-		t.Error("Could not retrieve CDRExporter FixedWidth config instance")
-	}
-}

config/config_test.go

@@ -1,6 +1,6 @@
 /*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -19,16 +19,14 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package config
 
 import (
-	"fmt"
 	"reflect"
 	"testing"
-	"time"
-
-	"github.com/cgrates/cgrates/utils"
 )
 
+var cfg *CGRConfig
+
 func TestConfigSharing(t *testing.T) {
-	cfg, _ := NewDefaultCGRConfig()
+	cfg, _ = NewDefaultCGRConfig()
 	SetCgrConfig(cfg)
 	cfgReturn := CgrConfig()
 	if !reflect.DeepEqual(cfgReturn, cfg) {
@@ -36,274 +34,27 @@ func TestConfigSharing(t *testing.T) {
 	}
 }
 
-// Make sure defaults did not change by mistake
-func TestDefaults(t *testing.T) {
-	cfg := &CGRConfig{}
-	errSet := cfg.setDefaults()
-	if errSet != nil {
-		t.Log(fmt.Sprintf("Coud not set defaults: %s!", errSet.Error()))
-		t.FailNow()
-	}
-	eCfg := &CGRConfig{}
-	eCfg.RatingDBType = REDIS
-	eCfg.RatingDBHost = "127.0.0.1"
-	eCfg.RatingDBPort = "6379"
-	eCfg.RatingDBName = "10"
-	eCfg.RatingDBUser = ""
-	eCfg.RatingDBPass = ""
-	eCfg.AccountDBType = REDIS
-	eCfg.AccountDBHost = "127.0.0.1"
-	eCfg.AccountDBPort = "6379"
-	eCfg.AccountDBName = "11"
-	eCfg.AccountDBUser = ""
-	eCfg.AccountDBPass = ""
-	eCfg.StorDBType = utils.MYSQL
-	eCfg.StorDBHost = "localhost"
-	eCfg.StorDBPort = "3306"
-	eCfg.StorDBName = "cgrates"
-	eCfg.StorDBUser = "cgrates"
-	eCfg.StorDBPass = "CGRateS.org"
-	eCfg.DBDataEncoding = utils.MSGPACK
-	eCfg.RPCJSONListen = "127.0.0.1:2012"
-	eCfg.RPCGOBListen = "127.0.0.1:2013"
-	eCfg.HTTPListen = "127.0.0.1:2080"
-	eCfg.DefaultReqType = utils.RATED
-	eCfg.DefaultTOR = "call"
-	eCfg.DefaultTenant = "cgrates.org"
-	eCfg.DefaultSubject = "cgrates"
-	eCfg.RoundingMethod = utils.ROUNDING_MIDDLE
-	eCfg.RoundingDecimals = 4
-	eCfg.XmlCfgDocument = nil
-	eCfg.RaterEnabled = false
-	eCfg.RaterBalancer = ""
-	eCfg.BalancerEnabled = false
-	eCfg.SchedulerEnabled = false
-	eCfg.CDRSEnabled = false
-	eCfg.CDRSExtraFields = []*utils.RSRField{}
-	eCfg.CDRSMediator = ""
-	eCfg.CdreCdrFormat = "csv"
-	eCfg.CdreDir = "/var/log/cgrates/cdr/cdrexport/csv"
-	eCfg.CdrcEnabled = false
-	eCfg.CdrcCdrs = utils.INTERNAL
-	eCfg.CdrcCdrsMethod = "http_cgr"
-	eCfg.CdrcRunDelay = time.Duration(0)
-	eCfg.CdrcCdrType = "csv"
-	eCfg.CdrcCdrInDir = "/var/log/cgrates/cdr/cdrc/in"
-	eCfg.CdrcCdrOutDir = "/var/log/cgrates/cdr/cdrc/out"
-	eCfg.CdrcSourceId = "freeswitch_csv"
-	eCfg.CdrcAccIdField = "0"
-	eCfg.CdrcReqTypeField = "1"
-	eCfg.CdrcDirectionField = "2"
-	eCfg.CdrcTenantField = "3"
-	eCfg.CdrcTorField = "4"
-	eCfg.CdrcAccountField = "5"
-	eCfg.CdrcSubjectField = "6"
-	eCfg.CdrcDestinationField = "7"
-	eCfg.CdrcSetupTimeField = "8"
-	eCfg.CdrcAnswerTimeField = "9"
-	eCfg.CdrcDurationField = "10"
-	eCfg.CdrcExtraFields = []string{}
-	eCfg.MediatorEnabled = false
-	eCfg.MediatorRater = "internal"
-	eCfg.MediatorRaterReconnects = 3
-	eCfg.MediatorRunIds = []string{}
-	eCfg.MediatorSubjectFields = []string{}
-	eCfg.MediatorReqTypeFields = []string{}
-	eCfg.MediatorDirectionFields = []string{}
-	eCfg.MediatorTenantFields = []string{}
-	eCfg.MediatorTORFields = []string{}
-	eCfg.MediatorAccountFields = []string{}
-	eCfg.MediatorDestFields = []string{}
-	eCfg.MediatorSetupTimeFields = []string{}
-	eCfg.MediatorAnswerTimeFields = []string{}
-	eCfg.MediatorDurationFields = []string{}
-	eCfg.SMEnabled = false
-	eCfg.SMSwitchType = FS
-	eCfg.SMRater = "internal"
-	eCfg.SMRaterReconnects = 3
-	eCfg.SMDebitInterval = 10
-	eCfg.SMMaxCallDuration = time.Duration(3) * time.Hour
-	eCfg.SMRunIds = []string{}
-	eCfg.SMReqTypeFields = []string{}
-	eCfg.SMDirectionFields = []string{}
-	eCfg.SMTenantFields = []string{}
-	eCfg.SMTORFields = []string{}
-	eCfg.SMAccountFields = []string{}
-	eCfg.SMSubjectFields = []string{}
-	eCfg.SMDestFields = []string{}
-	eCfg.SMSetupTimeFields = []string{}
-	eCfg.SMAnswerTimeFields = []string{}
-	eCfg.SMDurationFields = []string{}
-	eCfg.FreeswitchServer = "127.0.0.1:8021"
-	eCfg.FreeswitchPass = "ClueCon"
-	eCfg.FreeswitchReconnects = 5
-	eCfg.HistoryAgentEnabled = false
-	eCfg.HistoryServer = "internal"
-	eCfg.HistoryServerEnabled = false
-	eCfg.HistoryDir = "/var/log/cgrates/history"
-	eCfg.HistorySaveInterval = time.Duration(1) * time.Second
-	eCfg.MailerServer = "localhost:25"
-	eCfg.MailerAuthUser = "cgrates"
-	eCfg.MailerAuthPass = "CGRateS.org"
-	eCfg.MailerFromAddr = "cgr-mailer@localhost.localdomain"
-	eCfg.CdreExportedFields = []*utils.RSRField{
-		&utils.RSRField{Id: utils.CGRID},
-		&utils.RSRField{Id: utils.MEDI_RUNID},
-		&utils.RSRField{Id: utils.ACCID},
-		&utils.RSRField{Id: utils.CDRHOST},
-		&utils.RSRField{Id: utils.REQTYPE},
-		&utils.RSRField{Id: utils.DIRECTION},
-		&utils.RSRField{Id: utils.TENANT},
-		&utils.RSRField{Id: utils.TOR},
-		&utils.RSRField{Id: utils.ACCOUNT},
-		&utils.RSRField{Id: utils.SUBJECT},
-		&utils.RSRField{Id: utils.DESTINATION},
-		&utils.RSRField{Id: utils.SETUP_TIME},
-		&utils.RSRField{Id: utils.ANSWER_TIME},
-		&utils.RSRField{Id: utils.DURATION},
-		&utils.RSRField{Id: utils.COST},
-	}
-	if !reflect.DeepEqual(cfg, eCfg) {
-		t.Log(eCfg)
-		t.Log(cfg)
-		t.Error("Defaults different than expected!")
-	}
-}
+func TestLoadCgrCfgWithDefaults(t *testing.T) {
+	JSN_CFG := `
+{
+"sm_freeswitch": {
+	"enabled": true,				// starts SessionManager service: <true|false>
+	"connections":[					// instantiate connections to multiple FreeSWITCH servers
+		{"server": "1.2.3.4:8021", "password": "ClueCon", "reconnects": 3},
+		{"server": "1.2.3.5:8021", "password": "ClueCon", "reconnects": 5}
+	],
+},
 
-func TestSanityCheck(t *testing.T) {
-	cfg := &CGRConfig{}
-	errSet := cfg.setDefaults()
-	if errSet != nil {
-		t.Error("Coud not set defaults: ", errSet.Error())
+}`
+	eCgrCfg, _ := NewDefaultCGRConfig()
+	eCgrCfg.SmFsConfig.Enabled = true
+	eCgrCfg.SmFsConfig.Connections = []*FsConnConfig{
+		&FsConnConfig{Server: "1.2.3.4:8021", Password: "ClueCon", Reconnects: 3},
+		&FsConnConfig{Server: "1.2.3.5:8021", Password: "ClueCon", Reconnects: 5},
 	}
-	if err := cfg.checkConfigSanity(); err != nil {
-		t.Error("Invalid defaults: ", err)
-	}
-	cfg.SMSubjectFields = []string{"sample1", "sample2", "sample3"}
-	if err := cfg.checkConfigSanity(); err == nil {
-		t.Error("Failed to detect config insanity")
-	}
-	cfg = &CGRConfig{}
-	cfg.CdreCdrFormat = utils.CDRE_FIXED_WIDTH
-	if err := cfg.checkConfigSanity(); err == nil {
-		t.Error("Failed to detect fixed_width dependency on xml configuration")
-	}
-}
-
-// Load config from file and make sure we have all set
-func TestConfigFromFile(t *testing.T) {
-	cfgPth := "test_data.txt"
-	cfg, err := NewCGRConfig(&cfgPth)
-	if err != nil {
-		t.Log(fmt.Sprintf("Could not parse config: %s!", err))
-		t.FailNow()
-	}
-	eCfg := &CGRConfig{} // Instance we expect to get out after reading config file
-	eCfg.setDefaults()
-	eCfg.RatingDBType = "test"
-	eCfg.RatingDBHost = "test"
-	eCfg.RatingDBPort = "test"
-	eCfg.RatingDBName = "test"
-	eCfg.RatingDBUser = "test"
-	eCfg.RatingDBPass = "test"
-	eCfg.AccountDBType = "test"
-	eCfg.AccountDBHost = "test"
-	eCfg.AccountDBPort = "test"
-	eCfg.AccountDBName = "test"
-	eCfg.AccountDBUser = "test"
-	eCfg.AccountDBPass = "test"
-	eCfg.StorDBType = "test"
-	eCfg.StorDBHost = "test"
-	eCfg.StorDBPort = "test"
-	eCfg.StorDBName = "test"
-	eCfg.StorDBUser = "test"
-	eCfg.StorDBPass = "test"
-	eCfg.DBDataEncoding = "test"
-	eCfg.RPCJSONListen = "test"
-	eCfg.RPCGOBListen = "test"
-	eCfg.HTTPListen = "test"
-	eCfg.DefaultReqType = "test"
-	eCfg.DefaultTOR = "test"
-	eCfg.DefaultTenant = "test"
-	eCfg.DefaultSubject = "test"
-	eCfg.RoundingMethod = "test"
-	eCfg.RoundingDecimals = 99
-	eCfg.RaterEnabled = true
-	eCfg.RaterBalancer = "test"
-	eCfg.BalancerEnabled = true
-	eCfg.SchedulerEnabled = true
-	eCfg.CDRSEnabled = true
-	eCfg.CDRSExtraFields = []*utils.RSRField{&utils.RSRField{Id: "test"}}
-	eCfg.CDRSMediator = "test"
-	eCfg.CdreCdrFormat = "test"
-	eCfg.CdreExportedFields = []*utils.RSRField{&utils.RSRField{Id: "test"}}
-	eCfg.CdreDir = "test"
-	eCfg.CdrcEnabled = true
-	eCfg.CdrcCdrs = "test"
-	eCfg.CdrcCdrsMethod = "test"
-	eCfg.CdrcRunDelay = time.Duration(99) * time.Second
-	eCfg.CdrcCdrType = "test"
-	eCfg.CdrcCdrInDir = "test"
-	eCfg.CdrcCdrOutDir = "test"
-	eCfg.CdrcSourceId = "test"
-	eCfg.CdrcAccIdField = "test"
-	eCfg.CdrcReqTypeField = "test"
-	eCfg.CdrcDirectionField = "test"
-	eCfg.CdrcTenantField = "test"
-	eCfg.CdrcTorField = "test"
-	eCfg.CdrcAccountField = "test"
-	eCfg.CdrcSubjectField = "test"
-	eCfg.CdrcDestinationField = "test"
-	eCfg.CdrcSetupTimeField = "test"
-	eCfg.CdrcAnswerTimeField = "test"
-	eCfg.CdrcDurationField = "test"
-	eCfg.CdrcExtraFields = []string{"test"}
-	eCfg.MediatorEnabled = true
-	eCfg.MediatorRater = "test"
-	eCfg.MediatorRaterReconnects = 99
-	eCfg.MediatorRunIds = []string{"test"}
-	eCfg.MediatorSubjectFields = []string{"test"}
-	eCfg.MediatorReqTypeFields = []string{"test"}
-	eCfg.MediatorDirectionFields = []string{"test"}
-	eCfg.MediatorTenantFields = []string{"test"}
-	eCfg.MediatorTORFields = []string{"test"}
-	eCfg.MediatorAccountFields = []string{"test"}
-	eCfg.MediatorDestFields = []string{"test"}
-	eCfg.MediatorSetupTimeFields = []string{"test"}
-	eCfg.MediatorAnswerTimeFields = []string{"test"}
-	eCfg.MediatorDurationFields = []string{"test"}
-	eCfg.SMEnabled = true
-	eCfg.SMSwitchType = "test"
-	eCfg.SMRater = "test"
-	eCfg.SMRaterReconnects = 99
-	eCfg.SMDebitInterval = 99
-	eCfg.SMMaxCallDuration = time.Duration(99) * time.Second
-	eCfg.SMRunIds = []string{"test"}
-	eCfg.SMReqTypeFields = []string{"test"}
-	eCfg.SMDirectionFields = []string{"test"}
-	eCfg.SMTenantFields = []string{"test"}
-	eCfg.SMTORFields = []string{"test"}
-	eCfg.SMAccountFields = []string{"test"}
-	eCfg.SMSubjectFields = []string{"test"}
-	eCfg.SMDestFields = []string{"test"}
-	eCfg.SMSetupTimeFields = []string{"test"}
-	eCfg.SMAnswerTimeFields = []string{"test"}
-	eCfg.SMDurationFields = []string{"test"}
-	eCfg.FreeswitchServer = "test"
-	eCfg.FreeswitchPass = "test"
-	eCfg.FreeswitchReconnects = 99
-	eCfg.HistoryAgentEnabled = true
-	eCfg.HistoryServer = "test"
-	eCfg.HistoryServerEnabled = true
-	eCfg.HistoryDir = "test"
-	eCfg.HistorySaveInterval = time.Duration(99) * time.Second
-	eCfg.MailerServer = "test"
-	eCfg.MailerAuthUser = "test"
-	eCfg.MailerAuthPass = "test"
-	eCfg.MailerFromAddr = "test"
-	if !reflect.DeepEqual(cfg, eCfg) {
-		t.Log(eCfg)
-		t.Log(cfg)
-		t.Error("Loading of configuration from file failed!")
+	if cgrCfg, err := NewCGRConfigFromJsonStringWithDefaults(JSN_CFG); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCgrCfg.SmFsConfig, cgrCfg.SmFsConfig) {
+		t.Errorf("Expected: %+v, received: %+v", eCgrCfg.SmFsConfig, cgrCfg.SmFsConfig)
 	}
 }

config/configcdrc_test.go

@@ -0,0 +1,194 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+func TestLoadCdrcConfigMultipleFiles(t *testing.T) {
+	cgrCfg, err := NewCGRConfigFromFolder(".")
+	if err != nil {
+		t.Error(err)
+	}
+	eCgrCfg, _ := NewDefaultCGRConfig()
+	eCgrCfg.CdrcProfiles = make(map[string]map[string]*CdrcConfig)
+	// Default instance first
+	eCgrCfg.CdrcProfiles["/var/log/cgrates/cdrc/in"] = map[string]*CdrcConfig{
+		"*default": &CdrcConfig{
+			Enabled:                 false,
+			Cdrs:                    "internal",
+			CdrFormat:               "csv",
+			FieldSeparator:          ',',
+			DataUsageMultiplyFactor: 1024,
+			RunDelay:                0,
+			MaxOpenFiles:            1024,
+			CdrInDir:                "/var/log/cgrates/cdrc/in",
+			CdrOutDir:               "/var/log/cgrates/cdrc/out",
+			FailedCallsPrefix:       "missed_calls",
+			CdrSourceId:             "freeswitch_csv",
+			CdrFilter:               utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP),
+			PartialRecordCache:      time.Duration(10) * time.Second,
+			HeaderFields:            make([]*CfgCdrField, 0),
+			ContentFields: []*CfgCdrField{
+				&CfgCdrField{Tag: "tor", Type: "cdrfield", CdrFieldId: "tor", Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "accid", Type: "cdrfield", CdrFieldId: "accid", Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "reqtype", Type: "cdrfield", CdrFieldId: "reqtype", Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "direction", Type: "cdrfield", CdrFieldId: "direction", Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "tenant", Type: "cdrfield", CdrFieldId: "tenant", Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "category", Type: "cdrfield", CdrFieldId: "category", Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "account", Type: "cdrfield", CdrFieldId: "account", Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "subject", Type: "cdrfield", CdrFieldId: "subject", Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: utils.ParseRSRFieldsMustCompile("10", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "setup_time", Type: "cdrfield", CdrFieldId: "setup_time", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "answer_time", Type: "cdrfield", CdrFieldId: "answer_time", Value: utils.ParseRSRFieldsMustCompile("12", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "usage", Type: "cdrfield", CdrFieldId: "usage", Value: utils.ParseRSRFieldsMustCompile("13", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+			},
+			TrailerFields: make([]*CfgCdrField, 0),
+		},
+	}
+	eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc1/in"] = map[string]*CdrcConfig{
+		"CDRC-CSV1": &CdrcConfig{
+			Enabled:                 true,
+			Cdrs:                    "internal",
+			CdrFormat:               "csv",
+			FieldSeparator:          ',',
+			DataUsageMultiplyFactor: 1024,
+			RunDelay:                0,
+			MaxOpenFiles:            1024,
+			CdrInDir:                "/tmp/cgrates/cdrc1/in",
+			CdrOutDir:               "/tmp/cgrates/cdrc1/out",
+			CdrSourceId:             "csv1",
+			CdrFilter:               utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP),
+			HeaderFields:            make([]*CfgCdrField, 0),
+			ContentFields: []*CfgCdrField{
+				&CfgCdrField{Tag: "tor", Type: "cdrfield", CdrFieldId: "tor", Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "accid", Type: "cdrfield", CdrFieldId: "accid", Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "reqtype", Type: "cdrfield", CdrFieldId: "reqtype", Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "direction", Type: "cdrfield", CdrFieldId: "direction", Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "tenant", Type: "cdrfield", CdrFieldId: "tenant", Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "category", Type: "cdrfield", CdrFieldId: "category", Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "account", Type: "cdrfield", CdrFieldId: "account", Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "subject", Type: "cdrfield", CdrFieldId: "subject", Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: utils.ParseRSRFieldsMustCompile("10", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "setup_time", Type: "cdrfield", CdrFieldId: "setup_time", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "answer_time", Type: "cdrfield", CdrFieldId: "answer_time", Value: utils.ParseRSRFieldsMustCompile("12", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "usage", Type: "cdrfield", CdrFieldId: "usage", Value: utils.ParseRSRFieldsMustCompile("13", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+			},
+			TrailerFields: make([]*CfgCdrField, 0),
+		},
+	}
+	eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc2/in"] = map[string]*CdrcConfig{
+		"CDRC-CSV2": &CdrcConfig{
+			Enabled:                 true,
+			Cdrs:                    "internal",
+			CdrFormat:               "csv",
+			FieldSeparator:          ',',
+			DataUsageMultiplyFactor: 0.000976563,
+			RunDelay:                0,
+			MaxOpenFiles:            1024,
+			CdrInDir:                "/tmp/cgrates/cdrc2/in",
+			CdrOutDir:               "/tmp/cgrates/cdrc2/out",
+			CdrSourceId:             "csv2",
+			CdrFilter:               utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP),
+			HeaderFields:            make([]*CfgCdrField, 0),
+			ContentFields: []*CfgCdrField{
+				&CfgCdrField{Tag: "", Type: "", CdrFieldId: "tor", Value: utils.ParseRSRFieldsMustCompile("~7:s/^(voice|data|sms|generic)$/*$1/", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: false},
+				&CfgCdrField{Tag: "", Type: "", CdrFieldId: "answer_time", Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: false},
+			},
+			TrailerFields: make([]*CfgCdrField, 0),
+		},
+	}
+	eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc3/in"] = map[string]*CdrcConfig{
+		"CDRC-CSV3": &CdrcConfig{
+			Enabled:                 true,
+			Cdrs:                    "internal",
+			CdrFormat:               "csv",
+			FieldSeparator:          ',',
+			DataUsageMultiplyFactor: 1024,
+			RunDelay:                0,
+			MaxOpenFiles:            1024,
+			CdrInDir:                "/tmp/cgrates/cdrc3/in",
+			CdrOutDir:               "/tmp/cgrates/cdrc3/out",
+			CdrSourceId:             "csv3",
+			CdrFilter:               utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP),
+			HeaderFields:            make([]*CfgCdrField, 0),
+			ContentFields: []*CfgCdrField{
+				&CfgCdrField{Tag: "tor", Type: "cdrfield", CdrFieldId: "tor", Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "accid", Type: "cdrfield", CdrFieldId: "accid", Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "reqtype", Type: "cdrfield", CdrFieldId: "reqtype", Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "direction", Type: "cdrfield", CdrFieldId: "direction", Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "tenant", Type: "cdrfield", CdrFieldId: "tenant", Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "category", Type: "cdrfield", CdrFieldId: "category", Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "account", Type: "cdrfield", CdrFieldId: "account", Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "subject", Type: "cdrfield", CdrFieldId: "subject", Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: utils.ParseRSRFieldsMustCompile("10", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "setup_time", Type: "cdrfield", CdrFieldId: "setup_time", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "answer_time", Type: "cdrfield", CdrFieldId: "answer_time", Value: utils.ParseRSRFieldsMustCompile("12", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "usage", Type: "cdrfield", CdrFieldId: "usage", Value: utils.ParseRSRFieldsMustCompile("13", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+			},
+			TrailerFields: make([]*CfgCdrField, 0),
+		},
+	}
+	if !reflect.DeepEqual(eCgrCfg.CdrcProfiles, cgrCfg.CdrcProfiles) {
+		t.Errorf("Expected: %+v, received: %+v", eCgrCfg.CdrcProfiles, cgrCfg.CdrcProfiles)
+	}
+}

config/helpers.go

@@ -1,66 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package config
-
-import (
-	"code.google.com/p/goconf/conf"
-	"errors"
-	"strings"
-
-	"github.com/cgrates/cgrates/utils"
-)
-
-// Adds support for slice values in config
-func ConfigSlice(c *conf.ConfigFile, section, valName string) ([]string, error) {
-	sliceStr, errGet := c.GetString(section, valName)
-	if errGet != nil {
-		return nil, errGet
-	}
-	cfgValStrs := strings.Split(sliceStr, ",")       // If need arrises, we can make the separator configurable
-	if len(cfgValStrs) == 1 && cfgValStrs[0] == "" { // Prevents returning iterable with empty value
-		return []string{}, nil
-	}
-	for _, elm := range cfgValStrs {
-		if elm == "" { //One empty element is presented when splitting empty string
-			return nil, errors.New("Empty values in config slice")
-
-		}
-	}
-	return cfgValStrs, nil
-}
-
-func ParseRSRFields(configVal string) ([]*utils.RSRField, error) {
-	cfgValStrs := strings.Split(configVal, string(utils.CSV_SEP))
-	if len(cfgValStrs) == 1 && cfgValStrs[0] == "" { // Prevents returning iterable with empty value
-		return []*utils.RSRField{}, nil
-	}
-	rsrFields := make([]*utils.RSRField, len(cfgValStrs))
-	for idx, cfgValStr := range cfgValStrs {
-		if len(cfgValStr) == 0 { //One empty element is presented when splitting empty string
-			return nil, errors.New("Empty values in config slice")
-
-		}
-		if rsrField, err := utils.NewRSRField(cfgValStr); err != nil {
-			return nil, err
-		} else {
-			rsrFields[idx] = rsrField
-		}
-	}
-	return rsrFields, nil
-}

config/libconfig.go

@@ -1,6 +1,6 @@
 /*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -19,21 +19,12 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package config
 
 import (
-	"reflect"
-	"regexp"
-	"testing"
-
 	"github.com/cgrates/cgrates/utils"
 )
 
-func TestParseRSRFields(t *testing.T) {
-	fields := `host,~sip_redirected_to:s/sip:\+49(\d+)@/0$1/,destination`
-	expectParsedFields := []*utils.RSRField{&utils.RSRField{Id: "host"},
-		&utils.RSRField{Id: "sip_redirected_to", RSRule: &utils.ReSearchReplace{regexp.MustCompile(`sip:\+49(\d+)@`), "0$1"}},
-		&utils.RSRField{Id: "destination"}}
-	if parsedFields, err := ParseRSRFields(fields); err != nil {
-		t.Error("Unexpected error: ", err.Error())
-	} else if !reflect.DeepEqual(parsedFields, expectParsedFields) {
-		t.Errorf("Unexpected value of parsed fields")
-	}
+type CdrReplicationCfg struct {
+	Transport   string
+	Server      string
+	Synchronous bool
+	CdrFilter   utils.RSRFields // Only replicate if the filters here are matching
 }