Correcting tutorial for opensips installation

fixes #185, fixes #183

.gitignore

@@ -9,3 +9,8 @@ a.out
 docs/_*
 bin
 .idea
+dean*
+data/vagrant/.vagrant
+data/vagrant/vagrant_ansible_inventory_default
+data/tutorials/fs_evsock/freeswitch/etc/freeswitch/
+vendor

.travis.yml

@@ -1,6 +1,10 @@
 language: go
 
-before_install: sudo apt-get -q install bzr
+go:
+  - 1.4
+  - 1.5
+
+script: $TRAVIS_BUILD_DIR/test.sh
 
 branches:
   only: master
@@ -12,8 +16,5 @@ notifications:
     on_success: change
     on_failure: always
   email:
-    on_success: change
+    on_success: never
     on_failure: always
-
-after_script:
-    ./build.sh && ./test.sh

CONTRIBUTING.md

@@ -0,0 +1,26 @@
+## Contributor Agreement (required for pull requests)
+
+While the CGRateS project is open source under the GPLv3 license, contributions
+back to this project which contain code must be made available to ITsysCOM GmbH under the terms of the MIT license.
+
+### What does this mean?
+
+This means ITsysCOM GmbH can do the following with code contributed
+back to the project from the open source community.
+
+1. Include the code in future CGRateS releases.
+2. Include the code, in original or modified form, in other products created by ITsysCOM GmbH, **including closed-source products**.
+
+### How do I agree to these terms?
+
+To agree to these terms, sign the contributing agreement in `CONTRIBUTORS.md` and
+include the change as a commit with your pull request. **Pull requests which contain
+commits from users not explicitly listed in the `CONTRIBUTORS.md` file will be
+rejected *without review*.**
+
+* You only need to sign the `CONTRIBUTORS.md` file prior to the first time you
+  submit a pull request (or include it as part of the pull request itself).
+* The submitter of a pull request which adds a user to the `CONTRIBUTORS.md` file
+  *must* match the user information which was added to the `CONTRIBUTORS.md` file.
+  For example, only @danbogos can send a pull request containing a commit which adds
+  @danbogos to the `CONTRIBUTORS.md` file.

CONTRIBUTORS.md

@@ -0,0 +1,42 @@
+## Contributors
+
+By signing this agreement, the undersigned agree that contributions made to the CGRateS
+project are provided under the terms of the MIT license, included below. For additional
+information, please see the [`CONTRIBUTING.md`](CONTRIBUTING.md) file.
+
+**Pull requests which add a user to this file may only be submitted by the user who was added to the file.**
+
+### Contributor License Text
+
+> The MIT License (MIT)
+>
+> Copyright © <year> <copyright holders>
+>
+> Permission is hereby granted, free of charge, to any person obtaining a copy
+> of this software and associated documentation files (the "Software"), to deal
+> in the Software without restriction, including without limitation the rights
+> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+> copies of the Software, and to permit persons to whom the Software is
+> furnished to do so, subject to the following conditions:
+>
+> The above copyright notice and this permission notice shall be included in
+> all copies or substantial portions of the Software.
+>
+> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+> THE SOFTWARE.
+
+### Undersigned
+
+| GitHub Username | Full Name |
+| --------------- | --------- |
+| @danbogos | Dan Christian Bogos |
+| @eloycoto | Eloy Coto Pereiro |
+
+<!-- to sign, include a single line above this comment containing the following text:
+| @username | First Last |
+-->

README.md

@@ -1,6 +1,8 @@
-# Rating system for Telecom & ISP environments #
+## Real-time Charging System for Telecom & ISP environments ##
 
-## Features ##
+[![Build Status](https://secure.travis-ci.org/cgrates/cgrates.png)](http://travis-ci.org/cgrates/cgrates)
+
+### Features ###
 + Rates for prepaid and for postpaid
 + The budget expressed in money and/or minutes (seconds)
 + High accuracy rating: configurable to milliseconds
@@ -11,13 +13,16 @@
 + Good documentation
 + Commercial support available
 
-## Documentation ##
-Browsable HTML http://readthedocs.org/docs/cgrates/
+### Documentation ###
+[Step by steps tutorials](https://cgrates.readthedocs.org/en/latest/tut_freeswitch.html)
+
+[Debian apt-get repository](https://cgrates.readthedocs.org/en/latest/tut_freeswitch_installs.html#cgrates)
+
+Browsable HTML docs http://readthedocs.org/docs/cgrates/
 
 PDF, Epub, Manpage http://readthedocs.org/projects/cgrates/downloads/
 
-API reference http://gopkgdoc.appspot.com/pkg/github.com/cgrates/cgrates
+API reference [godoc](http://godoc.org/github.com/cgrates/cgrates/apier)
 
-Also check irc.freenode.net#cgrates and [Google group](https://groups.google.com/forum/#!forum/cgrates) for a more real-time support.
+Also check [irc.freenode.net #cgrates](irc://irc.freenode.net:6667/cgrates)  ([Webchat](http://webchat.freenode.net?randomnick=1&channels=%23cgrates)) and [Google group](https://groups.google.com/forum/#!forum/cgrates) for a more real-time support.
 
-Continous integration: [![Build Status](https://goci.herokuapp.com/project/image/github.com/cgrates/cgrates "Continous integration")](http://goci.me/project/github.com/cgrates/cgrates) [![Build Status](https://secure.travis-ci.org/cgrates/cgrates.png)](http://travis-ci.org/cgrates/cgrates)

apier/v1/accounts.go

@@ -0,0 +1,286 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"math"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type AttrAcntAction struct {
+	Tenant    string
+	Account   string
+	Direction string
+}
+
+type AccountActionTiming struct {
+	ActionPlanId string    // The id of the ActionPlanId profile attached to the account
+	Uuid         string    // The id to reference this particular ActionTiming
+	ActionsId    string    // The id of actions which will be executed
+	NextExecTime time.Time // Next execution time
+}
+
+func (self *ApierV1) GetAccountActionPlan(attrs AttrAcntAction, reply *[]*AccountActionTiming) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Direction"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(strings.Join(missing, ","), "")
+	}
+	accountATs := make([]*AccountActionTiming, 0)
+	allATs, err := self.RatingDb.GetAllActionPlans()
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	for _, ats := range allATs {
+		for _, at := range ats {
+			if utils.IsSliceMember(at.AccountIds, utils.AccountKey(attrs.Tenant, attrs.Account, attrs.Direction)) {
+				accountATs = append(accountATs, &AccountActionTiming{Uuid: at.Uuid, ActionPlanId: at.Id, ActionsId: at.ActionsId, NextExecTime: at.GetNextStartTime(time.Now())})
+			}
+		}
+	}
+	*reply = accountATs
+	return nil
+}
+
+type AttrRemActionTiming struct {
+	ActionPlanId    string // Id identifying the ActionTimings profile
+	ActionTimingId  string // Internal CGR id identifying particular ActionTiming, *all for all user related ActionTimings to be canceled
+	Tenant          string // Tenant he account belongs to
+	Account         string // Account name
+	Direction       string // Traffic direction
+	ReloadScheduler bool   // If set it will reload the scheduler after adding
+}
+
+// Removes an ActionTimings or parts of it depending on filters being set
+func (self *ApierV1) RemActionTiming(attrs AttrRemActionTiming, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"ActionPlanId"}); len(missing) != 0 { // Only mandatory ActionPlanId
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if len(attrs.Account) != 0 { // Presence of Account requires complete account details to be provided
+		if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Direction"}); len(missing) != 0 {
+			return utils.NewErrMandatoryIeMissing(missing...)
+		}
+	}
+	_, err := engine.Guardian.Guard(func() (interface{}, error) {
+		ats, err := self.RatingDb.GetActionPlans(attrs.ActionPlanId)
+		if err != nil {
+			return 0, err
+		} else if len(ats) == 0 {
+			return 0, utils.ErrNotFound
+		}
+		ats = engine.RemActionPlan(ats, attrs.ActionTimingId, utils.AccountKey(attrs.Tenant, attrs.Account, attrs.Direction))
+		if err := self.RatingDb.SetActionPlans(attrs.ActionPlanId, ats); err != nil {
+			return 0, err
+		}
+		return 0, nil
+	}, 0, utils.ACTION_TIMING_PREFIX)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if attrs.ReloadScheduler && self.Sched != nil {
+		self.Sched.LoadActionPlans(self.RatingDb)
+		self.Sched.Restart()
+	}
+	*reply = OK
+	return nil
+}
+
+// Returns a list of ActionTriggers on an account
+func (self *ApierV1) GetAccountActionTriggers(attrs AttrAcntAction, reply *engine.ActionTriggers) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Direction"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if balance, err := self.AccountDb.GetAccount(utils.AccountKey(attrs.Tenant, attrs.Account, attrs.Direction)); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = balance.ActionTriggers
+	}
+	return nil
+}
+
+type AttrRemAcntActionTriggers struct {
+	Tenant           string // Tenant he account belongs to
+	Account          string // Account name
+	Direction        string // Traffic direction
+	ActionTriggersId string // Id filtering only specific id to remove (can be regexp pattern)
+}
+
+// Returns a list of ActionTriggers on an account
+func (self *ApierV1) RemAccountActionTriggers(attrs AttrRemAcntActionTriggers, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Direction"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	balanceId := utils.AccountKey(attrs.Tenant, attrs.Account, attrs.Direction)
+	_, err := engine.Guardian.Guard(func() (interface{}, error) {
+		ub, err := self.AccountDb.GetAccount(balanceId)
+		if err != nil {
+			return 0, err
+		}
+		nactrs := make(engine.ActionTriggers, 0)
+		for _, actr := range ub.ActionTriggers {
+			match, _ := regexp.MatchString(attrs.ActionTriggersId, actr.Id)
+			if len(attrs.ActionTriggersId) != 0 && !match {
+				nactrs = append(nactrs, actr)
+			}
+		}
+		ub.ActionTriggers = nactrs
+		if err := self.AccountDb.SetAccount(ub); err != nil {
+			return 0, err
+		}
+		return 0, nil
+	}, 0, balanceId)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = OK
+	return nil
+}
+
+// Ads a new account into dataDb. If already defined, returns success.
+func (self *ApierV1) SetAccount(attr utils.AttrSetAccount, reply *string) error {
+	if missing := utils.MissingStructFields(&attr, []string{"Tenant", "Direction", "Account"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	balanceId := utils.AccountKey(attr.Tenant, attr.Account, attr.Direction)
+	var ub *engine.Account
+	var ats engine.ActionPlans
+	_, err := engine.Guardian.Guard(func() (interface{}, error) {
+		if bal, _ := self.AccountDb.GetAccount(balanceId); bal != nil {
+			ub = bal
+		} else { // Not found in db, create it here
+			ub = &engine.Account{
+				Id: balanceId,
+			}
+		}
+
+		if len(attr.ActionPlanId) != 0 {
+			var err error
+			ats, err = self.RatingDb.GetActionPlans(attr.ActionPlanId)
+			if err != nil {
+				return 0, err
+			}
+			for _, at := range ats {
+				at.AccountIds = append(at.AccountIds, balanceId)
+			}
+		}
+		if len(attr.ActionTriggersId) != 0 {
+			atrs, err := self.RatingDb.GetActionTriggers(attr.ActionTriggersId)
+			if err != nil {
+				return 0, err
+			}
+			ub.ActionTriggers = atrs
+		}
+		if attr.AllowNegative != nil {
+			ub.AllowNegative = *attr.AllowNegative
+		}
+		if attr.Disabled != nil {
+			ub.Disabled = *attr.Disabled
+		}
+		// All prepared, save account
+		if err := self.AccountDb.SetAccount(ub); err != nil {
+			return 0, err
+		}
+		return 0, nil
+	}, 0, balanceId)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if len(ats) != 0 {
+		_, err := engine.Guardian.Guard(func() (interface{}, error) { // ToDo: Try locking it above on read somehow
+			if err := self.RatingDb.SetActionPlans(attr.ActionPlanId, ats); err != nil {
+				return 0, err
+			}
+			return 0, nil
+		}, 0, utils.ACTION_TIMING_PREFIX)
+		if err != nil {
+			return utils.NewErrServerError(err)
+		}
+		if self.Sched != nil {
+			self.Sched.LoadActionPlans(self.RatingDb)
+			self.Sched.Restart()
+		}
+	}
+	*reply = OK // This will mark saving of the account, error still can show up in actionTimingsId
+	return nil
+}
+
+func (self *ApierV1) RemoveAccount(attr utils.AttrRemoveAccount, reply *string) error {
+	if missing := utils.MissingStructFields(&attr, []string{"Tenant", "Direction", "Account"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	accountId := utils.AccountKey(attr.Tenant, attr.Account, attr.Direction)
+	_, err := engine.Guardian.Guard(func() (interface{}, error) {
+		if err := self.AccountDb.RemoveAccount(accountId); err != nil {
+			return 0, err
+		}
+		return 0, nil
+	}, 0, accountId)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+
+	*reply = OK
+	return nil
+}
+
+func (self *ApierV1) GetAccounts(attr utils.AttrGetAccounts, reply *[]*engine.Account) error {
+	if len(attr.Tenant) == 0 {
+		return utils.NewErrMandatoryIeMissing("Tenanat")
+	}
+	if len(attr.Direction) == 0 {
+		attr.Direction = utils.OUT
+	}
+	var accountKeys []string
+	var err error
+	if len(attr.AccountIds) == 0 {
+		if accountKeys, err = self.AccountDb.GetKeysForPrefix(utils.ACCOUNT_PREFIX + utils.ConcatenatedKey(attr.Direction, attr.Tenant)); err != nil {
+			return err
+		}
+	} else {
+		for _, acntId := range attr.AccountIds {
+			if len(acntId) == 0 { // Source of error returned from redis (key not found)
+				continue
+			}
+			accountKeys = append(accountKeys, utils.ACCOUNT_PREFIX+utils.ConcatenatedKey(attr.Direction, attr.Tenant, acntId))
+		}
+	}
+	if len(accountKeys) == 0 {
+		return nil
+	}
+	var limitedAccounts []string
+	if attr.Limit != 0 {
+		max := math.Min(float64(attr.Offset+attr.Limit), float64(len(accountKeys)))
+		limitedAccounts = accountKeys[attr.Offset:int(max)]
+	} else {
+		limitedAccounts = accountKeys[attr.Offset:]
+	}
+	retAccounts := make([]*engine.Account, 0)
+	for _, acntKey := range limitedAccounts {
+		if acnt, err := self.AccountDb.GetAccount(acntKey[len(utils.ACCOUNT_PREFIX):]); err != nil && err != utils.ErrNotFound { // Not found is not an error here
+			return err
+		} else if acnt != nil {
+			retAccounts = append(retAccounts, acnt)
+		}
+	}
+	*reply = retAccounts
+	return nil
+}

apier/v1/accounts_test.go

@@ -0,0 +1,106 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"testing"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var (
+	apierAcnts            *ApierV1
+	apierAcntsAcntStorage *engine.MapStorage
+)
+
+func init() {
+	apierAcntsAcntStorage, _ = engine.NewMapStorage()
+	cfg, _ := config.NewDefaultCGRConfig()
+	apierAcnts = &ApierV1{AccountDb: engine.AccountingStorage(apierAcntsAcntStorage), Config: cfg}
+}
+
+func TestSetAccounts(t *testing.T) {
+	cgrTenant := "cgrates.org"
+	iscTenant := "itsyscom.com"
+	b10 := &engine.Balance{Value: 10, Weight: 10}
+	cgrAcnt1 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, cgrTenant, "account1"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	cgrAcnt2 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, cgrTenant, "account2"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	cgrAcnt3 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, cgrTenant, "account3"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	iscAcnt1 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, iscTenant, "account1"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	iscAcnt2 := &engine.Account{Id: utils.ConcatenatedKey(utils.OUT, iscTenant, "account2"),
+		BalanceMap: map[string]engine.BalanceChain{utils.MONETARY + engine.OUTBOUND: engine.BalanceChain{b10}}}
+	for _, account := range []*engine.Account{cgrAcnt1, cgrAcnt2, cgrAcnt3, iscAcnt1, iscAcnt2} {
+		if err := apierAcntsAcntStorage.SetAccount(account); err != nil {
+			t.Error(err)
+		}
+	}
+	apierAcntsAcntStorage.CacheRatingPrefixes(utils.ACTION_PREFIX)
+}
+
+/*
+func TestGetAccountIds(t *testing.T) {
+	var accountIds []string
+	var attrs AttrGetAccountIds
+	if err := apierAcnts.GetAccountIds(attrs, &accountIds); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accountIds) != 5 {
+		t.Errorf("Accounts returned: %+v", accountIds)
+	}
+}
+*/
+
+func TestGetAccounts(t *testing.T) {
+	var accounts []*engine.Account
+	var attrs utils.AttrGetAccounts
+	if err := apierAcnts.GetAccounts(utils.AttrGetAccounts{Tenant: "cgrates.org"}, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 3 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+	attrs = utils.AttrGetAccounts{Tenant: "itsyscom.com"}
+	if err := apierAcnts.GetAccounts(attrs, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 2 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+	attrs = utils.AttrGetAccounts{Tenant: "cgrates.org", AccountIds: []string{"account1"}}
+	if err := apierAcnts.GetAccounts(attrs, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 1 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+	attrs = utils.AttrGetAccounts{Tenant: "itsyscom.com", AccountIds: []string{"INVALID"}}
+	if err := apierAcnts.GetAccounts(attrs, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 0 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+	attrs = utils.AttrGetAccounts{Tenant: "INVALID"}
+	if err := apierAcnts.GetAccounts(attrs, &accounts); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if len(accounts) != 0 {
+		t.Errorf("Accounts returned: %+v", accounts)
+	}
+}

apier/v1/aliases.go

@@ -0,0 +1,140 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"errors"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type AttrAddRatingSubjectAliases struct {
+	Tenant, Category, Subject string
+	Aliases                   []string
+}
+
+type AttrAddAccountAliases struct {
+	Tenant, Category, Account string
+	Aliases                   []string
+}
+
+// Add aliases configured for a rating profile subject <Deprecated>
+func (self *ApierV1) AddRatingSubjectAliases(attrs AttrAddRatingSubjectAliases, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Subject", "Aliases"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if attrs.Category == "" {
+		attrs.Category = utils.CALL
+	}
+	aliases := engine.GetAliasService()
+	if aliases == nil {
+		return errors.New("ALIASES_NOT_ENABLED")
+	}
+	var ignr string
+	for _, alias := range attrs.Aliases {
+		if err := aliases.SetAlias(
+			engine.Alias{Direction: utils.META_OUT, Tenant: attrs.Tenant, Category: attrs.Category, Account: alias, Subject: alias, Context: utils.ALIAS_CONTEXT_RATING,
+				Values: engine.AliasValues{&engine.AliasValue{DestinationId: utils.META_ANY,
+					Pairs: engine.AliasPairs{"Subject": map[string]string{alias: attrs.Subject}}, Weight: 10.0}}}, &ignr); err != nil {
+			return utils.NewErrServerError(err)
+		}
+	}
+	*reply = utils.OK
+	return nil
+}
+
+// Remove aliases configured for a rating profile subject
+func (self *ApierV1) RemRatingSubjectAliases(tenantRatingSubject engine.TenantRatingSubject, reply *string) error {
+	if missing := utils.MissingStructFields(&tenantRatingSubject, []string{"Tenant", "Subject"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	aliases := engine.GetAliasService()
+	if aliases == nil {
+		return errors.New("ALIASES_NOT_ENABLED")
+	}
+	var reverseAliases map[string][]*engine.Alias
+	if err := aliases.GetReverseAlias(engine.AttrReverseAlias{Target: "Subject", Alias: tenantRatingSubject.Subject, Context: utils.ALIAS_CONTEXT_RATING}, &reverseAliases); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	var ignr string
+	for _, aliass := range reverseAliases {
+		for _, alias := range aliass {
+			if alias.Tenant != tenantRatingSubject.Tenant {
+				continue // From another tenant
+			}
+			if err := aliases.RemoveAlias(*alias, &ignr); err != nil {
+				return utils.NewErrServerError(err)
+			}
+		}
+	}
+	*reply = utils.OK
+	return nil
+}
+
+func (self *ApierV1) AddAccountAliases(attrs AttrAddAccountAliases, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Account", "Aliases"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if attrs.Category == "" {
+		attrs.Category = utils.CALL
+	}
+	aliases := engine.GetAliasService()
+	if aliases == nil {
+		return errors.New("ALIASES_NOT_ENABLED")
+	}
+	var ignr string
+	for _, alias := range attrs.Aliases {
+		if err := aliases.SetAlias(
+			engine.Alias{Direction: utils.META_OUT, Tenant: attrs.Tenant, Category: attrs.Category, Account: alias, Subject: alias, Context: utils.ALIAS_CONTEXT_RATING,
+				Values: engine.AliasValues{&engine.AliasValue{DestinationId: utils.META_ANY,
+					Pairs: engine.AliasPairs{"Account": map[string]string{alias: attrs.Account}}, Weight: 10.0}}}, &ignr); err != nil {
+			return utils.NewErrServerError(err)
+		}
+	}
+	*reply = utils.OK
+	return nil
+}
+
+// Remove aliases configured for an account
+func (self *ApierV1) RemAccountAliases(tenantAccount engine.TenantAccount, reply *string) error {
+	if missing := utils.MissingStructFields(&tenantAccount, []string{"Tenant", "Account"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	aliases := engine.GetAliasService()
+	if aliases == nil {
+		return errors.New("ALIASES_NOT_ENABLED")
+	}
+	var reverseAliases map[string][]*engine.Alias
+	if err := aliases.GetReverseAlias(engine.AttrReverseAlias{Target: "Account", Alias: tenantAccount.Account, Context: utils.ALIAS_CONTEXT_RATING}, &reverseAliases); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	var ignr string
+	for _, aliass := range reverseAliases {
+		for _, alias := range aliass {
+			if alias.Tenant != tenantAccount.Tenant {
+				continue // From another tenant
+			}
+			if err := aliases.RemoveAlias(*alias, &ignr); err != nil {
+				return utils.NewErrServerError(err)
+			}
+		}
+	}
+	*reply = utils.OK
+	return nil
+}

apier/v1/auth.go

@@ -0,0 +1,73 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Returns MaxUsage (for calls in seconds), -1 for no limit
+func (self *ApierV1) GetMaxUsage(usageRecord engine.UsageRecord, maxUsage *float64) error {
+	err := engine.LoadUserProfile(&usageRecord, "ExtraFields")
+	if err != nil {
+		return err
+	}
+	if usageRecord.TOR == "" {
+		usageRecord.TOR = utils.VOICE
+	}
+	if usageRecord.ReqType == "" {
+		usageRecord.ReqType = self.Config.DefaultReqType
+	}
+	if usageRecord.Direction == "" {
+		usageRecord.Direction = utils.OUT
+	}
+	if usageRecord.Tenant == "" {
+		usageRecord.Tenant = self.Config.DefaultTenant
+	}
+	if usageRecord.Category == "" {
+		usageRecord.Category = self.Config.DefaultCategory
+	}
+	if usageRecord.Subject == "" {
+		usageRecord.Subject = usageRecord.Account
+	}
+	if usageRecord.SetupTime == "" {
+		usageRecord.SetupTime = utils.META_NOW
+	}
+	if usageRecord.Usage == "" {
+		usageRecord.Usage = strconv.FormatFloat(self.Config.MaxCallDuration.Seconds(), 'f', -1, 64)
+	}
+	storedCdr, err := usageRecord.AsStoredCdr(self.Config.DefaultTimezone)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	var maxDur float64
+	if err := self.Responder.GetDerivedMaxSessionTime(storedCdr, &maxDur); err != nil {
+		return err
+	}
+	if maxDur == -1.0 {
+		*maxUsage = -1.0
+		return nil
+	}
+	*maxUsage = time.Duration(maxDur).Seconds()
+	return nil
+}

apier/v1/cdrc.go

@@ -0,0 +1,43 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type AttrReloadConfig struct {
+	ConfigDir string
+}
+
+// Retrieves the callCost out of CGR logDb
+func (apier *ApierV1) ReloadCdrcConfig(attrs AttrReloadConfig, reply *string) error {
+	if attrs.ConfigDir == "" {
+		attrs.ConfigDir = utils.CONFIG_DIR
+	}
+	newCfg, err := config.NewCGRConfigFromFolder(attrs.ConfigDir)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	apier.Config.CdrcProfiles = newCfg.CdrcProfiles // ToDo: Check if there can be any concurency involved here so we need to lock maybe
+	apier.Config.ConfigReloads[utils.CDRC] <- struct{}{}
+	*reply = OK
+	return nil
+}

apier/v1/cdre.go

@@ -0,0 +1,230 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/cgrates/cgrates/cdre"
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func (self *ApierV1) ExportCdrsToZipString(attr utils.AttrExpFileCdrs, reply *string) error {
+	tmpDir := "/tmp"
+	attr.ExportDir = &tmpDir // Enforce exporting to tmp always so we avoid cleanup issues
+	efc := utils.ExportedFileCdrs{}
+	if err := self.ExportCdrsToFile(attr, &efc); err != nil {
+		return err
+	} else if efc.TotalRecords == 0 || len(efc.ExportedFilePath) == 0 {
+		return errors.New("No CDR records to export")
+	}
+	// Create a buffer to write our archive to.
+	buf := new(bytes.Buffer)
+	// Create a new zip archive.
+	w := zip.NewWriter(buf)
+	// read generated file
+	content, err := ioutil.ReadFile(efc.ExportedFilePath)
+	if err != nil {
+		return err
+	}
+	exportFileName := path.Base(efc.ExportedFilePath)
+	f, err := w.Create(exportFileName)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(content)
+	if err != nil {
+		return err
+	}
+	// Write metadata into a separate file with extension .cgr
+	medaData, err := json.MarshalIndent(efc, "", "  ")
+	if err != nil {
+		errors.New("Failed creating metadata content")
+	}
+	medatadaFileName := exportFileName[:len(path.Ext(exportFileName))] + ".cgr"
+	mf, err := w.Create(medatadaFileName)
+	if err != nil {
+		return err
+	}
+	_, err = mf.Write(medaData)
+	if err != nil {
+		return err
+	}
+	// Make sure to check the error on Close.
+	if err := w.Close(); err != nil {
+		return err
+	}
+	if err := os.Remove(efc.ExportedFilePath); err != nil {
+		fmt.Errorf("Failed removing exported file at path: %s", efc.ExportedFilePath)
+	}
+	*reply = base64.StdEncoding.EncodeToString(buf.Bytes())
+	return nil
+}
+
+// Export Cdrs to file
+func (self *ApierV1) ExportCdrsToFile(attr utils.AttrExpFileCdrs, reply *utils.ExportedFileCdrs) error {
+	var err error
+
+	cdreReloadStruct := <-self.Config.ConfigReloads[utils.CDRE]                  // Read the content of the channel, locking it
+	defer func() { self.Config.ConfigReloads[utils.CDRE] <- cdreReloadStruct }() // Unlock reloads at exit
+	exportTemplate := self.Config.CdreProfiles[utils.META_DEFAULT]
+	if attr.ExportTemplate != nil && len(*attr.ExportTemplate) != 0 { // Export template prefered, use it
+		var hasIt bool
+		if exportTemplate, hasIt = self.Config.CdreProfiles[*attr.ExportTemplate]; !hasIt {
+			return fmt.Errorf("%s:ExportTemplate", utils.ErrNotFound.Error())
+		}
+	}
+	if exportTemplate == nil {
+		return fmt.Errorf("%s:ExportTemplate", utils.ErrMandatoryIeMissing.Error())
+	}
+	cdrFormat := exportTemplate.CdrFormat
+	if attr.CdrFormat != nil && len(*attr.CdrFormat) != 0 {
+		cdrFormat = strings.ToLower(*attr.CdrFormat)
+	}
+	if !utils.IsSliceMember(utils.CdreCdrFormats, cdrFormat) {
+		return fmt.Errorf("%s:%s", utils.ErrMandatoryIeMissing.Error(), "CdrFormat")
+	}
+	fieldSep := exportTemplate.FieldSeparator
+	if attr.FieldSeparator != nil && len(*attr.FieldSeparator) != 0 {
+		fieldSep, _ = utf8.DecodeRuneInString(*attr.FieldSeparator)
+		if fieldSep == utf8.RuneError {
+			return fmt.Errorf("%s:FieldSeparator:%s", utils.ErrServerError.Error(), "Invalid")
+		}
+	}
+	exportDir := exportTemplate.ExportDir
+	if attr.ExportDir != nil && len(*attr.ExportDir) != 0 {
+		exportDir = *attr.ExportDir
+	}
+	exportId := strconv.FormatInt(time.Now().Unix(), 10)
+	if attr.ExportId != nil && len(*attr.ExportId) != 0 {
+		exportId = *attr.ExportId
+	}
+	fileName := fmt.Sprintf("cdre_%s.%s", exportId, cdrFormat)
+	if attr.ExportFileName != nil && len(*attr.ExportFileName) != 0 {
+		fileName = *attr.ExportFileName
+	}
+	filePath := path.Join(exportDir, fileName)
+	if cdrFormat == utils.DRYRUN {
+		filePath = utils.DRYRUN
+	}
+	dataUsageMultiplyFactor := exportTemplate.DataUsageMultiplyFactor
+	if attr.DataUsageMultiplyFactor != nil && *attr.DataUsageMultiplyFactor != 0.0 {
+		dataUsageMultiplyFactor = *attr.DataUsageMultiplyFactor
+	}
+	smsUsageMultiplyFactor := exportTemplate.SmsUsageMultiplyFactor
+	if attr.SmsUsageMultiplyFactor != nil && *attr.SmsUsageMultiplyFactor != 0.0 {
+		smsUsageMultiplyFactor = *attr.SmsUsageMultiplyFactor
+	}
+	genericUsageMultiplyFactor := exportTemplate.GenericUsageMultiplyFactor
+	if attr.GenericUsageMultiplyFactor != nil && *attr.GenericUsageMultiplyFactor != 0.0 {
+		genericUsageMultiplyFactor = *attr.GenericUsageMultiplyFactor
+	}
+	costMultiplyFactor := exportTemplate.CostMultiplyFactor
+	if attr.CostMultiplyFactor != nil && *attr.CostMultiplyFactor != 0.0 {
+		costMultiplyFactor = *attr.CostMultiplyFactor
+	}
+	costShiftDigits := exportTemplate.CostShiftDigits
+	if attr.CostShiftDigits != nil {
+		costShiftDigits = *attr.CostShiftDigits
+	}
+	roundingDecimals := exportTemplate.CostRoundingDecimals
+	if attr.RoundDecimals != nil {
+		roundingDecimals = *attr.RoundDecimals
+	}
+	maskDestId := exportTemplate.MaskDestId
+	if attr.MaskDestinationId != nil && len(*attr.MaskDestinationId) != 0 {
+		maskDestId = *attr.MaskDestinationId
+	}
+	maskLen := exportTemplate.MaskLength
+	if attr.MaskLength != nil {
+		maskLen = *attr.MaskLength
+	}
+	cdrsFltr, err := attr.AsCdrsFilter(self.Config.DefaultTimezone)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	cdrs, _, err := self.CdrDb.GetStoredCdrs(cdrsFltr)
+	if err != nil {
+		return err
+	} else if len(cdrs) == 0 {
+		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
+		return nil
+	}
+	cdrexp, err := cdre.NewCdrExporter(cdrs, self.CdrDb, exportTemplate, cdrFormat, fieldSep, exportId, dataUsageMultiplyFactor, smsUsageMultiplyFactor, genericUsageMultiplyFactor,
+		costMultiplyFactor, costShiftDigits, roundingDecimals, self.Config.RoundingDecimals, maskDestId, maskLen, self.Config.HttpSkipTlsVerify, self.Config.DefaultTimezone)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if cdrexp.TotalExportedCdrs() == 0 {
+		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
+		return nil
+	}
+	if err := cdrexp.WriteToFile(filePath); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), TotalCost: cdrexp.TotalCost(), FirstOrderId: cdrexp.FirstOrderId(), LastOrderId: cdrexp.LastOrderId()}
+	if !attr.SuppressCgrIds {
+		reply.ExportedCgrIds = cdrexp.PositiveExports()
+		reply.UnexportedCgrIds = cdrexp.NegativeExports()
+	}
+	return nil
+}
+
+// Remove Cdrs out of CDR storage
+func (self *ApierV1) RemCdrs(attrs utils.AttrRemCdrs, reply *string) error {
+	if len(attrs.CgrIds) == 0 {
+		return fmt.Errorf("%s:CgrIds", utils.ErrMandatoryIeMissing.Error())
+	}
+	if err := self.CdrDb.RemStoredCdrs(attrs.CgrIds); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+// Reloads CDRE configuration out of folder specified
+func (apier *ApierV1) ReloadCdreConfig(attrs AttrReloadConfig, reply *string) error {
+	if attrs.ConfigDir == "" {
+		attrs.ConfigDir = utils.CONFIG_DIR
+	}
+	newCfg, err := config.NewCGRConfigFromFolder(attrs.ConfigDir)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	cdreReloadStruct := <-apier.Config.ConfigReloads[utils.CDRE] // Get the CDRE reload channel                     // Read the content of the channel, locking it
+	apier.Config.CdreProfiles = newCfg.CdreProfiles
+	apier.Config.ConfigReloads[utils.CDRE] <- cdreReloadStruct // Unlock reloads
+	engine.Logger.Info("<CDRE> Configuration reloaded")
+	*reply = OK
+	return nil
+}

apier/v1/cdrs.go

@@ -0,0 +1,60 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Retrieves the callCost out of CGR logDb
+func (apier *ApierV1) GetCallCostLog(attrs utils.AttrGetCallCost, reply *engine.CallCost) error {
+	if attrs.CgrId == "" {
+		return utils.NewErrMandatoryIeMissing("CgrId")
+	}
+	if attrs.RunId == "" {
+		attrs.RunId = utils.META_DEFAULT
+	}
+	if cc, err := apier.CdrDb.GetCallCostLog(attrs.CgrId, "", attrs.RunId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if cc == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = *cc
+	}
+	return nil
+}
+
+// Retrieves CDRs based on the filters
+func (apier *ApierV1) GetCdrs(attrs utils.AttrGetCdrs, reply *[]*engine.ExternalCdr) error {
+	cdrsFltr, err := attrs.AsCdrsFilter(apier.Config.DefaultTimezone)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if cdrs, _, err := apier.CdrDb.GetStoredCdrs(cdrsFltr); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(cdrs) == 0 {
+		*reply = make([]*engine.ExternalCdr, 0)
+	} else {
+		for _, cdr := range cdrs {
+			*reply = append(*reply, cdr.AsExternalCdr())
+		}
+	}
+	return nil
+}

apier/v1/cdrstatsv1.go

@@ -0,0 +1,70 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"fmt"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Interact with Stats server
+type CDRStatsV1 struct {
+	CdrStats engine.StatsInterface
+}
+
+type AttrGetMetrics struct {
+	StatsQueueId string // Id of the stats instance queried
+}
+
+func (sts *CDRStatsV1) GetMetrics(attr AttrGetMetrics, reply *map[string]float64) error {
+	if len(attr.StatsQueueId) == 0 {
+		return fmt.Errorf("%s:StatsQueueId", utils.ErrMandatoryIeMissing.Error())
+	}
+	return sts.CdrStats.GetValues(attr.StatsQueueId, reply)
+}
+
+func (sts *CDRStatsV1) GetQueueIds(empty string, reply *[]string) error {
+	return sts.CdrStats.GetQueueIds(0, reply)
+}
+
+func (sts *CDRStatsV1) GetQueue(id string, sq *engine.StatsQueue) error {
+	return sts.CdrStats.GetQueue(id, sq)
+}
+
+func (sts *CDRStatsV1) GetQueueTriggers(id string, ats *engine.ActionTriggers) error {
+	return sts.CdrStats.GetQueueTriggers(id, ats)
+}
+
+func (sts *CDRStatsV1) ReloadQueues(attr utils.AttrCDRStatsReloadQueues, reply *string) error {
+	if err := sts.CdrStats.ReloadQueues(attr.StatsQueueIds, nil); err != nil {
+		return err
+	}
+	*reply = utils.OK
+	return nil
+}
+
+func (sts *CDRStatsV1) ResetQueues(attr utils.AttrCDRStatsReloadQueues, reply *string) error {
+	if err := sts.CdrStats.ResetQueues(attr.StatsQueueIds, nil); err != nil {
+		return err
+	}
+	*reply = utils.OK
+	return nil
+}

apier/v1/cdrstatsv1_local_test.go

@@ -0,0 +1,204 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"fmt"
+	"net/http"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"path"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var cdrstCfgPath string
+var cdrstCfg *config.CGRConfig
+var cdrstRpc *rpc.Client
+
+func TestCDRStatsLclLoadConfig(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrstCfgPath = path.Join(*dataDir, "conf", "samples", "cdrstats")
+	if cdrstCfg, err = config.NewCGRConfigFromFolder(cfgPath); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestCDRStatsLclInitDataDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitDataDb(cdrstCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCDRStatsLclStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if _, err := engine.StopStartEngine(cdrstCfgPath, 1000); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestCDRStatsLclRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrstRpc, err = jsonrpc.Dial("tcp", cdrstCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+func TestCDRStatsLclLoadTariffPlanFromFolder(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	reply := ""
+	// Simple test that command is executed without errors
+	attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tariffplans", "cdrstats")}
+	if err := cdrstRpc.Call("ApierV1.LoadTariffPlanFromFolder", attrs, &reply); err != nil {
+		t.Error("Got error on ApierV1.LoadTariffPlanFromFolder: ", err.Error())
+	} else if reply != "OK" {
+		t.Error("Calling ApierV1.LoadTariffPlanFromFolder got reply: ", reply)
+	}
+	time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups
+}
+
+func TestCDRStatsLclGetQueueIds2(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var queueIds []string
+	eQueueIds := []string{"CDRST3", "CDRST4"}
+	if err := cdrstRpc.Call("CDRStatsV1.GetQueueIds", "", &queueIds); err != nil {
+		t.Error("Calling CDRStatsV1.GetQueueIds, got error: ", err.Error())
+	} else if len(eQueueIds) != len(queueIds) {
+		t.Errorf("Expecting: %v, received: %v", eQueueIds, queueIds)
+	}
+}
+
+func TestCDRStatsLclPostCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	httpClient := new(http.Client)
+	storedCdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsafa", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(),
+			AnswerTime: time.Now(), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
+			Cost: 1.01, RatedAccount: "dan", RatedSubject: "dan",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsafb", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(),
+			AnswerTime: time.Now(), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(5) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dan",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsafc", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), AnswerTime: time.Now(),
+			MediationRunId: utils.DEFAULT_RUNID,
+			Usage:          time.Duration(30) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dan",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsafd", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1", CdrSource: "test",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "+4986517174963", SetupTime: time.Now(), AnswerTime: time.Time{},
+			MediationRunId: utils.DEFAULT_RUNID,
+			Usage:          time.Duration(0) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01, RatedAccount: "dan", RatedSubject: "dan",
+		},
+	}
+	for _, storedCdr := range storedCdrs {
+		if _, err := httpClient.PostForm(fmt.Sprintf("http://%s/cdr_http", "127.0.0.1:2080"), storedCdr.AsHttpForm()); err != nil {
+			t.Error(err.Error())
+		}
+	}
+	time.Sleep(time.Duration(*waitRater) * time.Millisecond)
+
+}
+
+func TestCDRStatsLclGetMetrics1(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var rcvMetrics2 map[string]float64
+	expectedMetrics2 := map[string]float64{"ASR": 75, "ACD": 15}
+	if err := cdrstRpc.Call("CDRStatsV1.GetMetrics", AttrGetMetrics{StatsQueueId: "CDRST4"}, &rcvMetrics2); err != nil {
+		t.Error("Calling CDRStatsV1.GetMetrics, got error: ", err.Error())
+	} else if !reflect.DeepEqual(expectedMetrics2, rcvMetrics2) {
+		t.Errorf("Expecting: %v, received: %v", expectedMetrics2, rcvMetrics2)
+	}
+}
+
+// Test stats persistence
+func TestCDRStatsLclStatsPersistence(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	time.Sleep(time.Duration(2) * time.Second) // Allow stats to be updated in dataDb
+	if _, err := engine.StopStartEngine(cdrstCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+	var err error
+	cdrstRpc, err = jsonrpc.Dial("tcp", cdrstCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+	var rcvMetrics map[string]float64
+	expectedMetrics := map[string]float64{"ASR": 75, "ACD": 15}
+	if err := cdrstRpc.Call("CDRStatsV1.GetMetrics", AttrGetMetrics{StatsQueueId: "CDRST4"}, &rcvMetrics); err != nil {
+		t.Error("Calling CDRStatsV1.GetMetrics, got error: ", err.Error())
+	} else if !reflect.DeepEqual(expectedMetrics, rcvMetrics) {
+		t.Errorf("Expecting: %v, received: %v", expectedMetrics, rcvMetrics)
+	}
+}
+
+func TestCDRStatsLclResetMetrics(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	if err := cdrstRpc.Call("CDRStatsV1.ResetQueues", utils.AttrCDRStatsReloadQueues{StatsQueueIds: []string{"CDRST4"}}, &reply); err != nil {
+		t.Error("Calling CDRStatsV1.ResetQueues, got error: ", err.Error())
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply received: ", reply)
+	}
+	time.Sleep(time.Duration(*waitRater) * time.Millisecond)
+	var rcvMetrics2 map[string]float64
+	expectedMetrics2 := map[string]float64{"ASR": -1, "ACD": -1}
+	if err := cdrstRpc.Call("CDRStatsV1.GetMetrics", AttrGetMetrics{StatsQueueId: "CDRST4"}, &rcvMetrics2); err != nil {
+		t.Error("Calling CDRStatsV1.GetMetrics, got error: ", err.Error())
+	} else if !reflect.DeepEqual(expectedMetrics2, rcvMetrics2) {
+		t.Errorf("Expecting: %v, received: %v", expectedMetrics2, rcvMetrics2)
+	}
+}

apier/v1/cdrsv1.go

@@ -0,0 +1,82 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"time"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Receive CDRs via RPC methods
+type CdrsV1 struct {
+	CdrSrv *engine.CdrServer
+}
+
+// Designed for CGR internal usage
+func (self *CdrsV1) ProcessCdr(cdr *engine.StoredCdr, reply *string) error {
+	if err := self.CdrSrv.ProcessCdr(cdr); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
+}
+
+// Designed for external programs feeding CDRs to CGRateS
+func (self *CdrsV1) ProcessExternalCdr(cdr *engine.ExternalCdr, reply *string) error {
+	if err := self.CdrSrv.ProcessExternalCdr(cdr); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
+}
+
+// Remotely start mediation with specific runid, runs asynchronously, it's status will be displayed in syslog
+func (self *CdrsV1) RateCdrs(attrs utils.AttrRateCdrs, reply *string) error {
+	var tStart, tEnd time.Time
+	var err error
+	if len(attrs.TimeStart) != 0 {
+		if tStart, err = utils.ParseTimeDetectLayout(attrs.TimeStart, self.CdrSrv.Timezone()); err != nil {
+			return err
+		}
+	}
+	if len(attrs.TimeEnd) != 0 {
+		if tEnd, err = utils.ParseTimeDetectLayout(attrs.TimeEnd, self.CdrSrv.Timezone()); err != nil {
+			return err
+		}
+	}
+	//RateCdrs(cgrIds, runIds, tors, cdrHosts, cdrSources, reqTypes, directions, tenants, categories, accounts, subjects, destPrefixes []string,
+	//orderIdStart, orderIdEnd int64, timeStart, timeEnd time.Time, rerateErrors, rerateRated bool)
+	if err := self.CdrSrv.RateCdrs(attrs.CgrIds, attrs.MediationRunIds, attrs.TORs, attrs.CdrHosts, attrs.CdrSources, attrs.ReqTypes, attrs.Directions,
+		attrs.Tenants, attrs.Categories, attrs.Accounts, attrs.Subjects, attrs.DestinationPrefixes, attrs.RatedAccounts, attrs.RatedSubjects,
+		attrs.OrderIdStart, attrs.OrderIdEnd, tStart, tEnd, attrs.RerateErrors, attrs.RerateRated, attrs.SendToStats); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
+}
+
+func (self *CdrsV1) LogCallCost(ccl *engine.CallCostLog, reply *string) error {
+	if err := self.CdrSrv.LogCallCost(ccl); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
+}

apier/v1/costs.go

@@ -0,0 +1,59 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"time"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type AttrGetDataCost struct {
+	Direction                string
+	Category                 string
+	Tenant, Account, Subject string
+	StartTime                time.Time
+	Usage                    int64 // the call duration so far (till TimeEnd)
+}
+
+func (apier *ApierV1) GetDataCost(attrs AttrGetDataCost, reply *engine.DataCost) error {
+	usageAsDuration := time.Duration(attrs.Usage) * time.Second // Convert to seconds to match the loaded rates
+	cd := &engine.CallDescriptor{
+		Direction:     attrs.Direction,
+		Category:      attrs.Category,
+		Tenant:        attrs.Tenant,
+		Account:       attrs.Account,
+		Subject:       attrs.Subject,
+		TimeStart:     attrs.StartTime,
+		TimeEnd:       attrs.StartTime.Add(usageAsDuration),
+		DurationIndex: usageAsDuration,
+		TOR:           utils.DATA,
+	}
+	var cc engine.CallCost
+	if err := apier.Responder.GetCost(cd, &cc); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if dc, err := cc.ToDataCost(); err != nil {
+		return utils.NewErrServerError(err)
+	} else if dc != nil {
+		*reply = *dc
+	}
+	return nil
+}

apier/v1/debit.go

@@ -0,0 +1,66 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func (self *ApierV1) DebitUsage(usageRecord engine.UsageRecord, reply *string) error {
+	if missing := utils.MissingStructFields(&usageRecord, []string{"Account", "Destination", "Usage"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	err := engine.LoadUserProfile(usageRecord, "")
+	if err != nil {
+		*reply = err.Error()
+		return err
+	}
+	if usageRecord.TOR == "" {
+		usageRecord.TOR = utils.VOICE
+	}
+	if usageRecord.ReqType == "" {
+		usageRecord.ReqType = self.Config.DefaultReqType
+	}
+	if usageRecord.Direction == "" {
+		usageRecord.Direction = utils.OUT
+	}
+	if usageRecord.Tenant == "" {
+		usageRecord.Tenant = self.Config.DefaultTenant
+	}
+	if usageRecord.Category == "" {
+		usageRecord.Category = self.Config.DefaultCategory
+	}
+	if usageRecord.Subject == "" {
+		usageRecord.Subject = usageRecord.Account
+	}
+	if usageRecord.AnswerTime == "" {
+		usageRecord.AnswerTime = utils.META_NOW
+	}
+	cd, err := usageRecord.AsCallDescriptor(self.Config.DefaultTimezone)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	var cc engine.CallCost
+	if err := self.Responder.Debit(cd, &cc); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = OK
+	return nil
+}

apier/v1/derivedcharging.go

@@ -0,0 +1,117 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"fmt"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Get DerivedChargers applying to our call, appends general configured to account specific ones if that is configured
+func (self *ApierV1) GetDerivedChargers(attrs utils.AttrDerivedChargers, reply *utils.DerivedChargers) (err error) {
+	if missing := utils.MissingStructFields(&attrs, []string{"Tenant", "Direction", "Account", "Subject"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if hDc, err := engine.HandleGetDerivedChargers(self.RatingDb, &attrs); err != nil {
+		return utils.NewErrServerError(err)
+	} else if hDc != nil {
+		*reply = hDc
+	}
+	return nil
+}
+
+type AttrSetDerivedChargers struct {
+	Direction, Tenant, Category, Account, Subject string
+	DerivedChargers                               utils.DerivedChargers
+	Overwrite                                     bool // Do not overwrite if present in redis
+}
+
+func (self *ApierV1) SetDerivedChargers(attrs AttrSetDerivedChargers, reply *string) (err error) {
+	if len(attrs.Direction) == 0 {
+		attrs.Direction = utils.OUT
+	}
+	if len(attrs.Tenant) == 0 {
+		attrs.Tenant = utils.ANY
+	}
+	if len(attrs.Category) == 0 {
+		attrs.Category = utils.ANY
+	}
+	if len(attrs.Account) == 0 {
+		attrs.Account = utils.ANY
+	}
+	if len(attrs.Subject) == 0 {
+		attrs.Subject = utils.ANY
+	}
+	for _, dc := range attrs.DerivedChargers {
+		if _, err = utils.ParseRSRFields(dc.RunFilters, utils.INFIELD_SEP); err != nil { // Make sure rules are OK before loading in db
+			return fmt.Errorf("%s:%s", utils.ErrParserError.Error(), err.Error())
+		}
+	}
+	dcKey := utils.DerivedChargersKey(attrs.Direction, attrs.Tenant, attrs.Category, attrs.Account, attrs.Subject)
+	if !attrs.Overwrite {
+		if exists, err := self.RatingDb.HasData(utils.DESTINATION_PREFIX, dcKey); err != nil {
+			return utils.NewErrServerError(err)
+		} else if exists {
+			return utils.ErrExists
+		}
+	}
+	if err := self.RatingDb.SetDerivedChargers(dcKey, attrs.DerivedChargers); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if err := self.RatingDb.CacheRatingPrefixValues(map[string][]string{
+		utils.DERIVEDCHARGERS_PREFIX: []string{utils.DERIVEDCHARGERS_PREFIX + dcKey},
+	}); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.OK
+	return nil
+}
+
+type AttrRemDerivedChargers struct {
+	Direction, Tenant, Category, Account, Subject string
+}
+
+func (self *ApierV1) RemDerivedChargers(attrs AttrRemDerivedChargers, reply *string) error {
+	if len(attrs.Direction) == 0 {
+		attrs.Direction = utils.OUT
+	}
+	if len(attrs.Tenant) == 0 {
+		attrs.Tenant = utils.ANY
+	}
+	if len(attrs.Category) == 0 {
+		attrs.Category = utils.ANY
+	}
+	if len(attrs.Account) == 0 {
+		attrs.Account = utils.ANY
+	}
+	if len(attrs.Subject) == 0 {
+		attrs.Subject = utils.ANY
+	}
+	if err := self.RatingDb.SetDerivedChargers(utils.DerivedChargersKey(attrs.Direction, attrs.Tenant, attrs.Category, attrs.Account, attrs.Subject), nil); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	if err := self.RatingDb.CacheRatingPrefixes(utils.DERIVEDCHARGERS_PREFIX); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	return nil
+}

apier/v1/derivedcharging_test.go

@@ -0,0 +1,105 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var apierDcT *ApierV1
+
+func init() {
+	dataStorage, _ := engine.NewMapStorage()
+	cfg, _ := config.NewDefaultCGRConfig()
+	apierDcT = &ApierV1{RatingDb: engine.RatingStorage(dataStorage), Config: cfg}
+}
+
+/*
+func TestGetEmptyDC(t *testing.T) {
+	attrs := utils.AttrDerivedChargers{Tenant: "cgrates.org", Category: "call", Direction: "*out", Account: "dan", Subject: "dan"}
+	var dcs utils.DerivedChargers
+	if err := apierDcT.GetDerivedChargers(attrs, &dcs); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if !reflect.DeepEqual(dcs, apierDcT.Config.DerivedChargers) {
+		t.Error("Returned DerivedChargers not matching the configured ones")
+	}
+}
+*/
+
+func TestSetDC(t *testing.T) {
+	dcs1 := utils.DerivedChargers{
+		&utils.DerivedCharger{RunId: "extra1", ReqTypeField: "^prepaid", DirectionField: "*default", TenantField: "*default", CategoryField: "*default",
+			AccountField: "rif", SubjectField: "rif", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"},
+		&utils.DerivedCharger{RunId: "extra2", ReqTypeField: "*default", DirectionField: "*default", TenantField: "*default", CategoryField: "*default",
+			AccountField: "ivo", SubjectField: "ivo", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"},
+	}
+	attrs := AttrSetDerivedChargers{Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "dan", Subject: "dan", DerivedChargers: dcs1}
+	var reply string
+	if err := apierDcT.SetDerivedChargers(attrs, &reply); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply returned", reply)
+	}
+}
+
+func TestGetDC(t *testing.T) {
+	attrs := utils.AttrDerivedChargers{Tenant: "cgrates.org", Category: "call", Direction: "*out", Account: "dan", Subject: "dan"}
+	eDcs := utils.DerivedChargers{
+		&utils.DerivedCharger{RunId: "extra1", ReqTypeField: "^prepaid", DirectionField: "*default", TenantField: "*default", CategoryField: "*default",
+			AccountField: "rif", SubjectField: "rif", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"},
+		&utils.DerivedCharger{RunId: "extra2", ReqTypeField: "*default", DirectionField: "*default", TenantField: "*default", CategoryField: "*default",
+			AccountField: "ivo", SubjectField: "ivo", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"},
+	}
+	var dcs utils.DerivedChargers
+	if err := apierDcT.GetDerivedChargers(attrs, &dcs); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if !reflect.DeepEqual(dcs, eDcs) {
+		t.Errorf("Expecting: %v, received: %v", eDcs, dcs)
+	}
+}
+
+func TestRemDC(t *testing.T) {
+	attrs := AttrRemDerivedChargers{Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "dan", Subject: "dan"}
+	var reply string
+	if err := apierDcT.RemDerivedChargers(attrs, &reply); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if reply != utils.OK {
+		t.Error("Unexpected reply returned", reply)
+	}
+}
+
+/*
+func TestGetEmptyDC2(t *testing.T) {
+	attrs := utils.AttrDerivedChargers{Tenant: "cgrates.org", Category: "call", Direction: "*out", Account: "dan", Subject: "dan"}
+	var dcs utils.DerivedChargers
+	if err := apierDcT.GetDerivedChargers(attrs, &dcs); err != nil {
+		t.Error("Unexpected error", err.Error())
+	} else if !reflect.DeepEqual(dcs, apierDcT.Config.DerivedChargers) {
+		for _, dc := range dcs {
+			fmt.Printf("Got dc: %v\n", dc)
+		}
+		t.Error("Returned DerivedChargers not matching the configured ones")
+	}
+}
+*/

apier/v1/lcr.go

@@ -0,0 +1,83 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"fmt"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Computes the LCR for a specific request emulating a call
+func (self *ApierV1) GetLcr(lcrReq engine.LcrRequest, lcrReply *engine.LcrReply) error {
+	cd, err := lcrReq.AsCallDescriptor(self.Config.DefaultTimezone)
+	if err != nil {
+		return err
+	}
+	var lcrQried engine.LCRCost
+	if err := self.Responder.GetLCR(&engine.AttrGetLcr{CallDescriptor: cd, Paginator: lcrReq.Paginator}, &lcrQried); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if lcrQried.Entry == nil {
+		return utils.ErrNotFound
+	}
+	lcrReply.DestinationId = lcrQried.Entry.DestinationId
+	lcrReply.RPCategory = lcrQried.Entry.RPCategory
+	lcrReply.Strategy = lcrQried.Entry.Strategy
+	for _, qriedSuppl := range lcrQried.SupplierCosts {
+		if qriedSuppl.Error != "" {
+			engine.Logger.Err(fmt.Sprintf("LCR_ERROR: supplier <%s>, error <%s>", qriedSuppl.Supplier, qriedSuppl.Error))
+			if !lcrReq.IgnoreErrors {
+				return fmt.Errorf("%s:%s", utils.ErrServerError.Error(), "LCR_COMPUTE_ERRORS")
+			}
+			continue
+		}
+		if dtcs, err := utils.NewDTCSFromRPKey(qriedSuppl.Supplier); err != nil {
+			return utils.NewErrServerError(err)
+		} else {
+			lcrReply.Suppliers = append(lcrReply.Suppliers, &engine.LcrSupplier{Supplier: dtcs.Subject, Cost: qriedSuppl.Cost, QOS: qriedSuppl.QOS})
+		}
+	}
+	return nil
+}
+
+// Computes the LCR for a specific request emulating a call, returns a comma separated list of suppliers
+func (self *ApierV1) GetLcrSuppliers(lcrReq engine.LcrRequest, suppliers *string) (err error) {
+	cd, err := lcrReq.AsCallDescriptor(self.Config.DefaultTimezone)
+	if err != nil {
+		return err
+	}
+	var lcrQried engine.LCRCost
+	if err := self.Responder.GetLCR(&engine.AttrGetLcr{CallDescriptor: cd, Paginator: lcrReq.Paginator}, &lcrQried); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if lcrQried.HasErrors() {
+		lcrQried.LogErrors()
+		if !lcrReq.IgnoreErrors {
+			return fmt.Errorf("%s:%s", utils.ErrServerError.Error(), "LCR_COMPUTE_ERRORS")
+		}
+	}
+	if suppliersStr, err := lcrQried.SuppliersString(); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*suppliers = suppliersStr
+	}
+	return nil
+}

apier/v1/mediator.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -16,10 +16,4 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
 
-package cgrates
-
-import (
-	_ "github.com/cgrates/cgrates/cmd/cgr-console"
-	_ "github.com/cgrates/cgrates/cmd/cgr-loader"
-	_ "github.com/cgrates/cgrates/cmd/cgr-rater"
-)
+package v1

apier/v1/scheduler.go

@@ -0,0 +1,171 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"errors"
+	"strings"
+	"time"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+/*
+[
+    {
+        u'ActionsId': u'BONUS_1',
+        u'Uuid': u'5b5ba53b40b1d44380cce52379ec5c0d',
+        u'Weight': 10,
+        u'Timing': {
+            u'Timing': {
+                u'MonthDays': [
+
+                ],
+                u'Months': [
+
+                ],
+                u'WeekDays': [
+
+                ],
+                u'Years': [
+                    2013
+                ],
+                u'StartTime': u'11: 00: 00',
+                u'EndTime': u''
+            },
+            u'Rating': None,
+            u'Weight': 0
+        },
+        u'AccountIds': [
+            u'*out: cgrates.org: 1001',
+            u'*out: cgrates.org: 1002',
+            u'*out: cgrates.org: 1003',
+            u'*out: cgrates.org: 1004',
+            u'*out: cgrates.org: 1005'
+        ],
+        u'Id': u'PREPAID_10'
+    },
+    {
+        u'ActionsId': u'PREPAID_10',
+        u'Uuid': u'b16ab12740e2e6c380ff7660e8b55528',
+        u'Weight': 10,
+        u'Timing': {
+            u'Timing': {
+                u'MonthDays': [
+
+                ],
+                u'Months': [
+
+                ],
+                u'WeekDays': [
+
+                ],
+                u'Years': [
+                    2013
+                ],
+                u'StartTime': u'11: 00: 00',
+                u'EndTime': u''
+            },
+            u'Rating': None,
+            u'Weight': 0
+        },
+        u'AccountIds': [
+            u'*out: cgrates.org: 1001',
+            u'*out: cgrates.org: 1002',
+            u'*out: cgrates.org: 1003',
+            u'*out: cgrates.org: 1004',
+            u'*out: cgrates.org: 1005'
+        ],
+        u'Id': u'PREPAID_10'
+    }
+]
+*/
+
+type AttrsGetScheduledActions struct {
+	Direction, Tenant, Account string
+	TimeStart, TimeEnd         time.Time // Filter based on next runTime
+	utils.Paginator
+}
+
+type ScheduledActions struct {
+	NextRunTime                             time.Time
+	Accounts                                []*utils.DirectionTenantAccount
+	ActionsId, ActionPlanId, ActionPlanUuid string
+}
+
+func (self *ApierV1) GetScheduledActions(attrs AttrsGetScheduledActions, reply *[]*ScheduledActions) error {
+	schedActions := make([]*ScheduledActions, 0)
+	if self.Sched == nil {
+		return errors.New("SCHEDULER_NOT_ENABLED")
+	}
+	scheduledActions := self.Sched.GetQueue()
+	var min, max int
+	if attrs.Paginator.Offset != nil {
+		min = *attrs.Paginator.Offset
+	}
+	if attrs.Paginator.Limit != nil {
+		max = *attrs.Paginator.Limit
+	}
+	if max > len(scheduledActions) {
+		max = len(scheduledActions)
+	}
+	scheduledActions = scheduledActions[min : min+max]
+	for _, qActions := range scheduledActions {
+		sas := &ScheduledActions{ActionsId: qActions.ActionsId, ActionPlanId: qActions.Id, ActionPlanUuid: qActions.Uuid}
+		if attrs.SearchTerm != "" &&
+			!(strings.Contains(sas.ActionPlanId, attrs.SearchTerm) ||
+				strings.Contains(sas.ActionsId, attrs.SearchTerm)) {
+			continue
+		}
+		sas.NextRunTime = qActions.GetNextStartTime(time.Now())
+		if !attrs.TimeStart.IsZero() && sas.NextRunTime.Before(attrs.TimeStart) {
+			continue // Filter here only requests in the filtered interval
+		}
+		if !attrs.TimeEnd.IsZero() && (sas.NextRunTime.After(attrs.TimeEnd) || sas.NextRunTime.Equal(attrs.TimeEnd)) {
+			continue
+		}
+		acntFiltersMatch := false
+		for _, acntKey := range qActions.AccountIds {
+			directionMatched := len(attrs.Direction) == 0
+			tenantMatched := len(attrs.Tenant) == 0
+			accountMatched := len(attrs.Account) == 0
+			dta, _ := utils.NewDTAFromAccountKey(acntKey)
+			sas.Accounts = append(sas.Accounts, dta)
+			// One member matching
+			if !directionMatched && attrs.Direction == dta.Direction {
+				directionMatched = true
+			}
+			if !tenantMatched && attrs.Tenant == dta.Tenant {
+				tenantMatched = true
+			}
+			if !accountMatched && attrs.Account == dta.Account {
+				accountMatched = true
+			}
+			if directionMatched && tenantMatched && accountMatched {
+				acntFiltersMatch = true
+			}
+		}
+		if !acntFiltersMatch {
+			continue
+		}
+		schedActions = append(schedActions, sas)
+	}
+	*reply = schedActions
+	return nil
+}

apier/v1/smv1.go

@@ -0,0 +1,47 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/sessionmanager"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Interact with SessionManager
+type SessionManagerV1 struct {
+	SMs []sessionmanager.SessionManager // List of session managers since we support having more than one active session manager running on one host
+}
+
+func (self *SessionManagerV1) ActiveSessionMangers(ignored string, reply *[]sessionmanager.SessionManager) error {
+	if len(self.SMs) == 0 {
+		return utils.ErrNotFound
+	}
+	*reply = self.SMs
+	return nil
+}
+
+func (self *SessionManagerV1) ActiveSessions(attrs utils.AttrGetSMASessions, reply *[]*sessionmanager.ActiveSession) error {
+	if attrs.SessionManagerIndex > len(self.SMs)-1 {
+		return utils.ErrNotFound
+	}
+	for _, session := range self.SMs[attrs.SessionManagerIndex].Sessions() {
+		*reply = append(*reply, session.AsActiveSessions()...)
+	}
+	return nil
+}

apier/v1/tp.go

@@ -0,0 +1,99 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+// Tariff plan related APIs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type AttrGetTPIds struct {
+}
+
+// Queries tarrif plan identities gathered from all tables.
+func (self *ApierV1) GetTPIds(attrs AttrGetTPIds, reply *[]string) error {
+	if ids, err := self.StorDb.GetTpIds(); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+type AttrImportTPZipFile struct {
+	TPid string
+	File []byte
+}
+
+func (self *ApierV1) ImportTPZipFile(attrs AttrImportTPZipFile, reply *string) error {
+	tmpDir, err := ioutil.TempDir("/tmp", "cgr_")
+	if err != nil {
+		*reply = "ERROR: creating temp directory!"
+		return err
+	}
+	zipFile := filepath.Join(tmpDir, "/file.zip")
+	if err = ioutil.WriteFile(zipFile, attrs.File, os.ModePerm); err != nil {
+		*reply = "ERROR: writing zip file!"
+		return err
+	}
+	if err = utils.Unzip(zipFile, tmpDir); err != nil {
+		*reply = "ERROR: unziping file!"
+		return err
+	}
+	csvfilesFound := false
+	if err = filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error {
+		if !info.IsDir() {
+			return nil
+		}
+		csvFiles, err := filepath.Glob(filepath.Join(path, "*csv"))
+		if csvFiles != nil {
+			if attrs.TPid == "" {
+				*reply = "ERROR: missing TPid!"
+				return err
+			}
+			csvImporter := engine.TPCSVImporter{
+				TPid:     attrs.TPid,
+				StorDb:   self.StorDb,
+				DirPath:  path,
+				Sep:      ',',
+				Verbose:  false,
+				ImportId: "",
+			}
+			if errImport := csvImporter.Run(); errImport != nil {
+				return errImport
+			}
+			csvfilesFound = true
+		}
+		return err
+	}); err != nil || !csvfilesFound {
+		*reply = "ERROR: finding csv files!"
+		return err
+	}
+	os.RemoveAll(tmpDir)
+	*reply = "OK"
+	return nil
+}

apier/v1/tpaccountactions.go

@@ -0,0 +1,166 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new AccountActions profile within a tariff plan
+func (self *ApierV1) SetTPAccountActions(attrs utils.TPAccountActions, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs,
+		[]string{"TPid", "LoadId", "Tenant", "Account", "Direction", "ActionPlanId", "ActionTriggersId"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	aas := engine.APItoModelAccountAction(&attrs)
+	if err := self.StorDb.SetTpAccountActions([]engine.TpAccountAction{*aas}); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPAccountActionsByLoadId struct {
+	TPid   string // Tariff plan id
+	LoadId string // AccountActions id
+}
+
+// Queries specific AccountActions profile on tariff plan
+func (self *ApierV1) GetTPAccountActionsByLoadId(attrs utils.TPAccountActions, reply *[]*utils.TPAccountActions) error {
+	mndtryFlds := []string{"TPid", "LoadId"}
+	if len(attrs.Account) != 0 { // If account provided as filter, make all related fields mandatory
+		mndtryFlds = append(mndtryFlds, "Tenant", "Account", "Direction")
+	}
+	if missing := utils.MissingStructFields(&attrs, mndtryFlds); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	aas := engine.APItoModelAccountAction(&attrs)
+	if aa, err := self.StorDb.GetTpAccountActions(aas); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(aa) == 0 {
+		return utils.ErrNotFound
+	} else {
+
+		tpAa, err := engine.TpAccountActions(aa).GetAccountActions()
+		if err != nil {
+			return err
+		}
+		var acts []*utils.TPAccountActions
+		if len(attrs.Account) != 0 {
+			acts = []*utils.TPAccountActions{tpAa[attrs.KeyId()]}
+		} else {
+			for _, actLst := range tpAa {
+				acts = append(acts, actLst)
+			}
+		}
+		*reply = acts
+	}
+	return nil
+}
+
+type AttrGetTPAccountActions struct {
+	TPid             string // Tariff plan id
+	AccountActionsId string // DerivedCharge id
+}
+
+// Queries specific DerivedCharge on tariff plan
+func (self *ApierV1) GetTPAccountActions(attrs AttrGetTPAccountActions, reply *utils.TPAccountActions) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "AccountActionsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpAa := &utils.TPAccountActions{TPid: attrs.TPid}
+	if err := tmpAa.SetAccountActionsId(attrs.AccountActionsId); err != nil {
+		return err
+	}
+	tmpAaa := engine.APItoModelAccountAction(tmpAa)
+	if aas, err := self.StorDb.GetTpAccountActions(tmpAaa); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(aas) == 0 {
+		return utils.ErrNotFound
+	} else {
+		tpAaa, err := engine.TpAccountActions(aas).GetAccountActions()
+		if err != nil {
+			return err
+		}
+		aa := tpAaa[tmpAa.KeyId()]
+		tpdc := utils.TPAccountActions{
+			TPid:             attrs.TPid,
+			ActionPlanId:     aa.ActionPlanId,
+			ActionTriggersId: aa.ActionTriggersId,
+		}
+		if err := tpdc.SetAccountActionsId(attrs.AccountActionsId); err != nil {
+			return err
+		}
+		*reply = tpdc
+	}
+	return nil
+}
+
+type AttrGetTPAccountActionIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries AccountActions identities on specific tariff plan.
+func (self *ApierV1) GetTPAccountActionLoadIds(attrs AttrGetTPAccountActionIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACCOUNT_ACTIONS, utils.TPDistinctIds{"loadid"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Queries DerivedCharges identities on specific tariff plan.
+func (self *ApierV1) GetTPAccountActionIds(attrs AttrGetTPAccountActionIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACCOUNT_ACTIONS, utils.TPDistinctIds{"loadid", "direction", "tenant", "account"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific AccountActions on Tariff plan
+func (self *ApierV1) RemTPAccountActions(attrs AttrGetTPAccountActions, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "Account", "Direction"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	aa := engine.TpAccountAction{Tpid: attrs.TPid}
+	if err := aa.SetAccountActionId(attrs.AccountActionsId); err != nil {
+		return err
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_ACCOUNT_ACTIONS, aa.Tpid, aa.Loadid, aa.Direction, aa.Tenant, aa.Account); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpactionplans.go

@@ -0,0 +1,107 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"fmt"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new ActionTimings profile within a tariff plan
+func (self *ApierV1) SetTPActionPlan(attrs utils.TPActionPlan, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionPlanId", "ActionPlan"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	for _, at := range attrs.ActionPlan {
+		requiredFields := []string{"ActionsId", "TimingId", "Weight"}
+		if missing := utils.MissingStructFields(at, requiredFields); len(missing) != 0 {
+			return fmt.Errorf("%s:Action:%s:%v", utils.ErrMandatoryIeMissing.Error(), at.ActionsId, missing)
+		}
+	}
+	ap := engine.APItoModelActionPlan(&attrs)
+	if err := self.StorDb.SetTpActionPlans(ap); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPActionPlan struct {
+	TPid string // Tariff plan id
+	Id   string // ActionPlans id
+}
+
+// Queries specific ActionPlan profile on tariff plan
+func (self *ApierV1) GetTPActionPlan(attrs AttrGetTPActionPlan, reply *utils.TPActionPlan) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Id"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ats, err := self.StorDb.GetTpActionPlans(attrs.TPid, attrs.Id); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(ats) == 0 {
+		return utils.ErrNotFound
+	} else { // Got the data we need, convert it
+		aps, err := engine.TpActionPlans(ats).GetActionPlans()
+		if err != nil {
+			return err
+		}
+		atRply := &utils.TPActionPlan{
+			TPid:         attrs.TPid,
+			ActionPlanId: attrs.Id,
+			ActionPlan:   aps[attrs.Id],
+		}
+		*reply = *atRply
+	}
+	return nil
+}
+
+type AttrGetTPActionPlanIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries ActionPlan identities on specific tariff plan.
+func (self *ApierV1) GetTPActionPlanIds(attrs AttrGetTPActionPlanIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACTION_PLANS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific ActionPlan on Tariff plan
+func (self *ApierV1) RemTPActionPlan(attrs AttrGetTPActionPlan, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Id"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_ACTION_PLANS, attrs.TPid, attrs.Id); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpactions.go

@@ -0,0 +1,105 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"fmt"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new Actions profile within a tariff plan
+func (self *ApierV1) SetTPActions(attrs utils.TPActions, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionsId", "Actions"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	for _, action := range attrs.Actions {
+		requiredFields := []string{"Identifier", "Weight"}
+		if action.BalanceType != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions
+			requiredFields = append(requiredFields, "Direction", "Units")
+		}
+		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
+			return fmt.Errorf("%s:Action:%s:%v", utils.ErrMandatoryIeMissing.Error(), action.Identifier, missing)
+		}
+	}
+	as := engine.APItoModelAction(&attrs)
+	if err := self.StorDb.SetTpActions(as); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPActions struct {
+	TPid      string // Tariff plan id
+	ActionsId string // Actions id
+}
+
+// Queries specific Actions profile on tariff plan
+func (self *ApierV1) GetTPActions(attrs AttrGetTPActions, reply *utils.TPActions) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if acts, err := self.StorDb.GetTpActions(attrs.TPid, attrs.ActionsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(acts) == 0 {
+		return utils.ErrNotFound
+	} else {
+		as, err := engine.TpActions(acts).GetActions()
+		if err != nil {
+
+		}
+		*reply = utils.TPActions{TPid: attrs.TPid, ActionsId: attrs.ActionsId, Actions: as[attrs.ActionsId]}
+	}
+	return nil
+}
+
+type AttrGetTPActionIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries Actions identities on specific tariff plan.
+func (self *ApierV1) GetTPActionIds(attrs AttrGetTPActionIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACTIONS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific Actions on Tariff plan
+func (self *ApierV1) RemTPActions(attrs AttrGetTPActions, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_ACTIONS, attrs.TPid, attrs.ActionsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpactiontriggers.go

@@ -0,0 +1,101 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new ActionTriggers profile within a tariff plan
+func (self *ApierV1) SetTPActionTriggers(attrs utils.TPActionTriggers, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs,
+		[]string{"TPid", "ActionTriggersId"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+
+	ats := engine.APItoModelActionTrigger(&attrs)
+	if err := self.StorDb.SetTpActionTriggers(ats); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPActionTriggers struct {
+	TPid             string // Tariff plan id
+	ActionTriggersId string // ActionTrigger id
+}
+
+// Queries specific ActionTriggers profile on tariff plan
+func (self *ApierV1) GetTPActionTriggers(attrs AttrGetTPActionTriggers, reply *utils.TPActionTriggers) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionTriggersId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ats, err := self.StorDb.GetTpActionTriggers(attrs.TPid, attrs.ActionTriggersId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(ats) == 0 {
+		return utils.ErrNotFound
+	} else {
+		atsMap, err := engine.TpActionTriggers(ats).GetActionTriggers()
+		if err != nil {
+			return err
+		}
+		atRply := &utils.TPActionTriggers{
+			TPid:             attrs.TPid,
+			ActionTriggersId: attrs.ActionTriggersId,
+			ActionTriggers:   atsMap[attrs.ActionTriggersId],
+		}
+		*reply = *atRply
+	}
+	return nil
+}
+
+type AttrGetTPActionTriggerIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries ActionTriggers identities on specific tariff plan.
+func (self *ApierV1) GetTPActionTriggerIds(attrs AttrGetTPActionTriggerIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ACTION_TRIGGERS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific ActionTriggers on Tariff plan
+func (self *ApierV1) RemTPActionTriggers(attrs AttrGetTPActionTriggers, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "ActionTriggersId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_ACTION_TRIGGERS, attrs.TPid, attrs.ActionTriggersId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpaliases.go

@@ -0,0 +1,96 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new alias within a tariff plan
+func (self *ApierV1) SetTPAlias(attrs utils.TPAliases, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Direction", "Tenant", "Category", "Account", "Subject", "Group"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tm := engine.APItoModelAliases(&attrs)
+	if err := self.StorDb.SetTpAliases(tm); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPAlias struct {
+	TPid    string // Tariff plan id
+	AliasId string
+}
+
+// Queries specific Alias on Tariff plan
+func (self *ApierV1) GetTPAlias(attr AttrGetTPAlias, reply *utils.TPAliases) error {
+	if missing := utils.MissingStructFields(&attr, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	al := &engine.TpAlias{Tpid: attr.TPid}
+	al.SetId(attr.AliasId)
+	if tms, err := self.StorDb.GetTpAliases(al); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(tms) == 0 {
+		return utils.ErrNotFound
+	} else {
+		tmMap, err := engine.TpAliases(tms).GetAliases()
+		if err != nil {
+			return err
+		}
+		*reply = *tmMap[al.GetId()]
+	}
+	return nil
+}
+
+type AttrGetTPAliasIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries alias identities on specific tariff plan.
+func (self *ApierV1) GetTPAliasIds(attrs AttrGetTPAliasIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_ALIASES, utils.TPDistinctIds{"`direction`", "`tenant`", "`category`", "`account`", "`subject`", "`group`"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific Alias on Tariff plan
+func (self *ApierV1) RemTPAlias(attrs AttrGetTPAlias, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "AliasId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_ALIASES, attrs.TPid); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpcdrstats.go

@@ -0,0 +1,103 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new CdrStats profile within a tariff plan
+func (self *ApierV1) SetTPCdrStats(attrs utils.TPCdrStats, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "CdrStatsId", "CdrStats"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	/*for _, action := range attrs.CdrStats {
+		requiredFields := []string{"Identifier", "Weight"}
+		if action.BalanceType != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions
+			requiredFields = append(requiredFields, "Direction", "Units")
+		}
+		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
+			return fmt.Errorf("%s:CdrStat:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
+		}
+	}*/
+	cs := engine.APItoModelCdrStat(&attrs)
+	if err := self.StorDb.SetTpCdrStats(cs); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPCdrStats struct {
+	TPid       string // Tariff plan id
+	CdrStatsId string // CdrStat id
+}
+
+// Queries specific CdrStat on tariff plan
+func (self *ApierV1) GetTPCdrStats(attrs AttrGetTPCdrStats, reply *utils.TPCdrStats) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "CdrStatsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if sgs, err := self.StorDb.GetTpCdrStats(attrs.TPid, attrs.CdrStatsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(sgs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		csMap, err := engine.TpCdrStats(sgs).GetCdrStats()
+		if err != nil {
+			return err
+		}
+		*reply = utils.TPCdrStats{TPid: attrs.TPid, CdrStatsId: attrs.CdrStatsId, CdrStats: csMap[attrs.CdrStatsId]}
+	}
+	return nil
+}
+
+type AttrGetTPCdrStatIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries CdrStats identities on specific tariff plan.
+func (self *ApierV1) GetTPCdrStatsIds(attrs AttrGetTPCdrStatIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_CDR_STATS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific CdrStats on Tariff plan
+func (self *ApierV1) RemTPCdrStats(attrs AttrGetTPCdrStats, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "CdrStatsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_SHARED_GROUPS, attrs.TPid, attrs.CdrStatsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpderivedcharges.go

@@ -0,0 +1,112 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new DerivedCharges profile within a tariff plan
+func (self *ApierV1) SetTPDerivedChargers(attrs utils.TPDerivedChargers, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Direction", "Tenant", "Category", "Account", "Subject"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	/*for _, action := range attrs.DerivedCharges {
+		requiredFields := []string{"Identifier", "Weight"}
+		if action.BalanceType != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions
+			requiredFields = append(requiredFields, "Direction", "Units")
+		}
+		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
+			return fmt.Errorf("%s:DerivedCharge:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
+		}
+	}*/
+	dc := engine.APItoModelDerivedCharger(&attrs)
+	if err := self.StorDb.SetTpDerivedChargers(dc); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPDerivedChargers struct {
+	TPid              string // Tariff plan id
+	DerivedChargersId string // DerivedCharge id
+}
+
+// Queries specific DerivedCharge on tariff plan
+func (self *ApierV1) GetTPDerivedChargers(attrs AttrGetTPDerivedChargers, reply *utils.TPDerivedChargers) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DerivedChargersId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpDc := &utils.TPDerivedChargers{TPid: attrs.TPid}
+	if err := tmpDc.SetDerivedChargersId(attrs.DerivedChargersId); err != nil {
+		return err
+	}
+	dcs := engine.APItoModelDerivedCharger(tmpDc)
+	if sgs, err := self.StorDb.GetTpDerivedChargers(&dcs[0]); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(sgs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		dcsMap, err := engine.TpDerivedChargers(dcs).GetDerivedChargers()
+		if err != nil {
+			return err
+		}
+		*reply = *dcsMap[attrs.DerivedChargersId]
+	}
+	return nil
+}
+
+type AttrGetTPDerivedChargeIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries DerivedCharges identities on specific tariff plan.
+func (self *ApierV1) GetTPDerivedChargerIds(attrs AttrGetTPDerivedChargeIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_DERIVED_CHARGERS, utils.TPDistinctIds{"loadid", "direction", "tenant", "category", "account", "subject"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific DerivedCharges on Tariff plan
+func (self *ApierV1) RemTPDerivedChargers(attrs AttrGetTPDerivedChargers, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DerivedChargesId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpDc := engine.TpDerivedCharger{}
+	if err := tmpDc.SetDerivedChargersId(attrs.DerivedChargersId); err != nil {
+		return err
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_DERIVED_CHARGERS, attrs.TPid, tmpDc.Loadid, tmpDc.Direction, tmpDc.Tenant, tmpDc.Category, tmpDc.Account, tmpDc.Subject); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpdestinationrates.go

@@ -0,0 +1,97 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+// This file deals with tp_destination_rates management over APIs
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new DestinationRate profile within a tariff plan
+func (self *ApierV1) SetTPDestinationRate(attrs utils.TPDestinationRate, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationRateId", "DestinationRates"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	drs := engine.APItoModelDestinationRate(&attrs)
+	if err := self.StorDb.SetTpDestinationRates(drs); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPDestinationRate struct {
+	TPid              string // Tariff plan id
+	DestinationRateId string // Rate id
+	utils.Paginator
+}
+
+// Queries specific DestinationRate profile on tariff plan
+func (self *ApierV1) GetTPDestinationRate(attrs AttrGetTPDestinationRate, reply *utils.TPDestinationRate) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationRateId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if drs, err := self.StorDb.GetTpDestinationRates(attrs.TPid, attrs.DestinationRateId, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(drs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		drsMap, err := engine.TpDestinationRates(drs).GetDestinationRates()
+		if err != nil {
+			return err
+		}
+		*reply = *drsMap[attrs.DestinationRateId]
+	}
+	return nil
+}
+
+type AttrTPDestinationRateIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries DestinationRate identities on specific tariff plan.
+func (self *ApierV1) GetTPDestinationRateIds(attrs AttrGetTPRateIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_DESTINATION_RATES, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific DestinationRate on Tariff plan
+func (self *ApierV1) RemTPDestinationRate(attrs AttrGetTPDestinationRate, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationRateId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_DESTINATION_RATES, attrs.TPid, attrs.DestinationRateId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpdestinations.go

@@ -0,0 +1,97 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new destination within a tariff plan
+func (self *ApierV1) SetTPDestination(attrs utils.TPDestination, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationId", "Prefixes"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	ds := engine.APItoModelDestination(&attrs)
+	if err := self.StorDb.SetTpDestinations(ds); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPDestination struct {
+	TPid          string // Tariff plan id
+	DestinationId string // Destination id
+}
+
+// Queries a specific destination
+func (self *ApierV1) GetTPDestination(attrs AttrGetTPDestination, reply *utils.TPDestination) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if storData, err := self.StorDb.GetTpDestinations(attrs.TPid, attrs.DestinationId); err != nil {
+
+		return utils.NewErrServerError(err)
+	} else if len(storData) == 0 {
+		return utils.ErrNotFound
+	} else {
+		dsts, err := engine.TpDestinations(storData).GetDestinations()
+		if err != nil {
+			return err
+		}
+		*reply = utils.TPDestination{
+			TPid:          attrs.TPid,
+			DestinationId: dsts[attrs.DestinationId].Id,
+			Prefixes:      dsts[attrs.DestinationId].Prefixes}
+	}
+	return nil
+}
+
+type AttrGetTPDestinationIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries destination identities on specific tariff plan.
+func (self *ApierV1) GetTPDestinationIds(attrs AttrGetTPDestinationIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_DESTINATIONS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+func (self *ApierV1) RemTPDestination(attrs AttrGetTPDestination, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "DestinationId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_DESTINATIONS, attrs.TPid, attrs.DestinationId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tplcrrules.go

@@ -0,0 +1,98 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new LcrRules profile within a tariff plan
+func (self *ApierV1) SetTPLcrRule(attrs utils.TPLcrRules, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Direction", "Tenant", "Category", "Account", "Subject"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tm := engine.APItoModelLcrRule(&attrs)
+	if err := self.StorDb.SetTpLCRs(tm); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPLcrRules struct {
+	TPid      string // Tariff plan id
+	LcrRuleId string // Lcr id
+}
+
+// Queries specific LcrRules profile on tariff plan
+func (self *ApierV1) GetTPLcrRule(attr AttrGetTPLcrRules, reply *utils.TPLcrRules) error {
+	if missing := utils.MissingStructFields(&attr, []string{"TPid", "LcrRuleId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	lcr := &engine.TpLcrRule{
+		Tpid: attr.TPid,
+	}
+	lcr.SetLcrRuleId(attr.LcrRuleId)
+	if lcrs, err := self.StorDb.GetTpLCRs(lcr); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(lcrs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		tmMap, err := engine.TpLcrRules(lcrs).GetLcrRules()
+		if err != nil {
+			return err
+		}
+		*reply = *tmMap[attr.LcrRuleId]
+	}
+	return nil
+}
+
+type AttrGetTPLcrIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries LcrRules identities on specific tariff plan.
+func (self *ApierV1) GetTPLcrRuleIds(attrs AttrGetTPLcrIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_LCRS, utils.TPDistinctIds{"direction", "tenant", "category", "account", "subject"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific LcrRules on Tariff plan
+func (self *ApierV1) RemTPLcrRule(attrs AttrGetTPLcrRules, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LcrRulesId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_LCRS, attrs.TPid, attrs.LcrRuleId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tprates.go

@@ -0,0 +1,96 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+// This file deals with tp_rates management over APIs
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new rate within a tariff plan
+func (self *ApierV1) SetTPRate(attrs utils.TPRate, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RateId", "RateSlots"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	r := engine.APItoModelRate(&attrs)
+	if err := self.StorDb.SetTpRates(r); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPRate struct {
+	TPid   string // Tariff plan id
+	RateId string // Rate id
+}
+
+// Queries specific Rate on tariff plan
+func (self *ApierV1) GetTPRate(attrs AttrGetTPRate, reply *utils.TPRate) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RateId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if rts, err := self.StorDb.GetTpRates(attrs.TPid, attrs.RateId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(rts) == 0 {
+		return utils.ErrNotFound
+	} else {
+		rtsMap, err := engine.TpRates(rts).GetRates()
+		if err != nil {
+			return err
+		}
+		*reply = *rtsMap[attrs.RateId]
+	}
+	return nil
+}
+
+type AttrGetTPRateIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries rate identities on specific tariff plan.
+func (self *ApierV1) GetTPRateIds(attrs AttrGetTPRateIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_RATES, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific Rate on Tariff plan
+func (self *ApierV1) RemTPRate(attrs AttrGetTPRate, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RateId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_RATES, attrs.TPid, attrs.RateId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpratingplans.go

@@ -0,0 +1,97 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+// This file deals with tp_destrates_timing management over APIs
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new DestinationRateTiming profile within a tariff plan
+func (self *ApierV1) SetTPRatingPlan(attrs utils.TPRatingPlan, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingPlanId", "RatingPlanBindings"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	rp := engine.APItoModelRatingPlan(&attrs)
+	if err := self.StorDb.SetTpRatingPlans(rp); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPRatingPlan struct {
+	TPid         string // Tariff plan id
+	RatingPlanId string // Rate id
+	utils.Paginator
+}
+
+// Queries specific RatingPlan profile on tariff plan
+func (self *ApierV1) GetTPRatingPlan(attrs AttrGetTPRatingPlan, reply *utils.TPRatingPlan) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingPlanId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if rps, err := self.StorDb.GetTpRatingPlans(attrs.TPid, attrs.RatingPlanId, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(rps) == 0 {
+		return utils.ErrNotFound
+	} else {
+		rpsMap, err := engine.TpRatingPlans(rps).GetRatingPlans()
+		if err != nil {
+			return err
+		}
+		*reply = utils.TPRatingPlan{TPid: attrs.TPid, RatingPlanId: attrs.RatingPlanId, RatingPlanBindings: rpsMap[attrs.RatingPlanId]}
+	}
+	return nil
+}
+
+type AttrGetTPRatingPlanIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries RatingPlan identities on specific tariff plan.
+func (self *ApierV1) GetTPRatingPlanIds(attrs AttrGetTPRatingPlanIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_RATING_PLANS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific RatingPlan on Tariff plan
+func (self *ApierV1) RemTPRatingPlan(attrs AttrGetTPRatingPlan, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingPlanId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_RATING_PLANS, attrs.TPid, attrs.RatingPlanId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpratingprofiles.go

@@ -0,0 +1,170 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+// This file deals with tp_rate_profiles management over APIs
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new RatingProfile within a tariff plan
+func (self *ApierV1) SetTPRatingProfile(attrs utils.TPRatingProfile, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "LoadId", "Tenant", "Category", "Direction", "Subject", "RatingPlanActivations"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	rpf := engine.APItoModelRatingProfile(&attrs)
+	if err := self.StorDb.SetTpRatingProfiles(rpf); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPRatingProfileByLoadId struct {
+	TPid   string // Tariff plan id
+	LoadId string // RatingProfile id
+}
+
+// Queries specific RatingProfile on tariff plan
+func (self *ApierV1) GetTPRatingProfilesByLoadId(attrs utils.TPRatingProfile, reply *[]*utils.TPRatingProfile) error {
+	mndtryFlds := []string{"TPid", "LoadId"}
+	if len(attrs.Subject) != 0 { // If Subject provided as filter, make all related fields mandatory
+		mndtryFlds = append(mndtryFlds, "Tenant", "TOR", "Direction", "Subject")
+	}
+	if missing := utils.MissingStructFields(&attrs, mndtryFlds); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	rpf := engine.APItoModelRatingProfile(&attrs)
+	if dr, err := self.StorDb.GetTpRatingProfiles(&rpf[0]); err != nil {
+		return utils.NewErrServerError(err)
+	} else if dr == nil {
+		return utils.ErrNotFound
+	} else {
+		rpfMap, err := engine.TpRatingProfiles(dr).GetRatingProfiles()
+		if err != nil {
+			return err
+		}
+		var rpfs []*utils.TPRatingProfile
+		if len(attrs.Subject) != 0 {
+			rpfs = []*utils.TPRatingProfile{rpfMap[attrs.KeyId()]}
+		} else {
+			for _, rpfLst := range rpfMap {
+				rpfs = append(rpfs, rpfLst)
+			}
+		}
+		*reply = rpfs
+	}
+	return nil
+}
+
+// Queries RatingProfile identities on specific tariff plan.
+func (self *ApierV1) GetTPRatingProfileLoadIds(attrs utils.AttrTPRatingProfileIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_RATE_PROFILES, utils.TPDistinctIds{"loadid"}, map[string]string{
+		"tenant":    attrs.Tenant,
+		"tor":       attrs.Category,
+		"direction": attrs.Direction,
+		"subject":   attrs.Subject,
+	}, new(utils.Paginator)); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+type AttrGetTPRatingProfile struct {
+	TPid            string // Tariff plan id
+	RatingProfileId string // RatingProfile id
+}
+
+// Queries specific RatingProfile on tariff plan
+func (self *ApierV1) GetTPRatingProfile(attrs AttrGetTPRatingProfile, reply *utils.TPRatingProfile) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingProfileId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpRpf := &utils.TPRatingProfile{TPid: attrs.TPid}
+	if err := tmpRpf.SetRatingProfilesId(attrs.RatingProfileId); err != nil {
+		return err
+	}
+	rpf := engine.APItoModelRatingProfile(tmpRpf)
+	if rpfs, err := self.StorDb.GetTpRatingProfiles(&rpf[0]); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(rpfs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		rpfMap, err := engine.TpRatingProfiles(rpfs).GetRatingProfiles()
+		if err != nil {
+			return err
+		}
+		rpf := rpfMap[tmpRpf.KeyId()]
+		tpdc := utils.TPRatingProfile{
+			TPid: attrs.TPid,
+			RatingPlanActivations: rpf.RatingPlanActivations,
+		}
+		if err := tpdc.SetRatingProfilesId(attrs.RatingProfileId); err != nil {
+			return err
+		}
+		*reply = tpdc
+	}
+	return nil
+}
+
+type AttrGetTPRatingProfileIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries RatingProfiles identities on specific tariff plan.
+func (self *ApierV1) GetTPRatingProfileIds(attrs AttrGetTPRatingProfileIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_RATE_PROFILES, utils.TPDistinctIds{"loadid", "direction", "tenant", "category", "subject"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific RatingProfiles on Tariff plan
+func (self *ApierV1) RemTPRatingProfile(attrs AttrGetTPRatingProfile, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "RatingProfileId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tmpRpf := engine.TpRatingProfile{}
+	if err := tmpRpf.SetRatingProfileId(attrs.RatingProfileId); err != nil {
+		return err
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_RATE_PROFILES, attrs.TPid, tmpRpf.Loadid, tmpRpf.Direction, tmpRpf.Tenant, tmpRpf.Category, tmpRpf.Subject); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpsharedgroups.go

@@ -0,0 +1,103 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new SharedGroups profile within a tariff plan
+func (self *ApierV1) SetTPSharedGroups(attrs utils.TPSharedGroups, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "SharedGroupsId", "SharedGroups"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	/*for _, action := range attrs.SharedGroups {
+		requiredFields := []string{"Identifier", "Weight"}
+		if action.BalanceType != "" { // Add some inter-dependent parameters - if balanceType then we are not talking about simply calling actions
+			requiredFields = append(requiredFields, "Direction", "Units")
+		}
+		if missing := utils.MissingStructFields(action, requiredFields); len(missing) != 0 {
+			return fmt.Errorf("%s:SharedGroup:%s:%v", utils.ERR_MANDATORY_IE_MISSING, action.Identifier, missing)
+		}
+	}*/
+	sg := engine.APItoModelSharedGroup(&attrs)
+	if err := self.StorDb.SetTpSharedGroups(sg); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPSharedGroups struct {
+	TPid           string // Tariff plan id
+	SharedGroupsId string // SharedGroup id
+}
+
+// Queries specific SharedGroup on tariff plan
+func (self *ApierV1) GetTPSharedGroups(attrs AttrGetTPSharedGroups, reply *utils.TPSharedGroups) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "SharedGroupsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if sgs, err := self.StorDb.GetTpSharedGroups(attrs.TPid, attrs.SharedGroupsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(sgs) == 0 {
+		return utils.ErrNotFound
+	} else {
+		sgMap, err := engine.TpSharedGroups(sgs).GetSharedGroups()
+		if err != nil {
+			return err
+		}
+		*reply = utils.TPSharedGroups{TPid: attrs.TPid, SharedGroupsId: attrs.SharedGroupsId, SharedGroups: sgMap[attrs.SharedGroupsId]}
+	}
+	return nil
+}
+
+type AttrGetTPSharedGroupIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries SharedGroups identities on specific tariff plan.
+func (self *ApierV1) GetTPSharedGroupIds(attrs AttrGetTPSharedGroupIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_SHARED_GROUPS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific SharedGroups on Tariff plan
+func (self *ApierV1) RemTPSharedGroups(attrs AttrGetTPSharedGroups, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "SharedGroupsId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_SHARED_GROUPS, attrs.TPid, attrs.SharedGroupsId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tptimings.go

@@ -0,0 +1,94 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new timing within a tariff plan
+func (self *ApierV1) SetTPTiming(attrs utils.ApierTPTiming, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "TimingId", "Years", "Months", "MonthDays", "WeekDays", "Time"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tm := engine.APItoModelTiming(&attrs)
+	if err := self.StorDb.SetTpTimings([]engine.TpTiming{*tm}); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPTiming struct {
+	TPid     string // Tariff plan id
+	TimingId string // Timing id
+}
+
+// Queries specific Timing on Tariff plan
+func (self *ApierV1) GetTPTiming(attrs AttrGetTPTiming, reply *utils.ApierTPTiming) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "TimingId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if tms, err := self.StorDb.GetTpTimings(attrs.TPid, attrs.TimingId); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(tms) == 0 {
+		return utils.ErrNotFound
+	} else {
+		tmMap, err := engine.TpTimings(tms).GetApierTimings()
+		if err != nil {
+			return err
+		}
+		*reply = *tmMap[attrs.TimingId]
+	}
+	return nil
+}
+
+type AttrGetTPTimingIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries timing identities on specific tariff plan.
+func (self *ApierV1) GetTPTimingIds(attrs AttrGetTPTimingIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_TIMINGS, utils.TPDistinctIds{"tag"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific Timing on Tariff plan
+func (self *ApierV1) RemTPTiming(attrs AttrGetTPTiming, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "TimingId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_TIMINGS, attrs.TPid, attrs.TimingId); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tpusers.go

@@ -0,0 +1,98 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v1
+
+import (
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Creates a new alias within a tariff plan
+func (self *ApierV1) SetTPUser(attrs utils.TPUsers, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "Direction", "Tenant", "Category", "Account", "Subject", "Group"}); len(missing) != 0 {
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	tm := engine.APItoModelUsers(&attrs)
+	if err := self.StorDb.SetTpUsers(tm); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = "OK"
+	return nil
+}
+
+type AttrGetTPUser struct {
+	TPid   string // Tariff plan id
+	UserId string
+}
+
+// Queries specific User on Tariff plan
+func (self *ApierV1) GetTPUser(attr AttrGetTPUser, reply *utils.TPUsers) error {
+	if missing := utils.MissingStructFields(&attr, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	usr := &engine.TpUser{
+		Tpid: attr.TPid,
+	}
+	usr.SetId(attr.UserId)
+	if tms, err := self.StorDb.GetTpUsers(usr); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(tms) == 0 {
+		return utils.ErrNotFound
+	} else {
+		tmMap, err := engine.TpUsers(tms).GetUsers()
+		if err != nil {
+			return err
+		}
+		*reply = *tmMap[usr.GetId()]
+	}
+	return nil
+}
+
+type AttrGetTPUserIds struct {
+	TPid string // Tariff plan id
+	utils.Paginator
+}
+
+// Queries alias identities on specific tariff plan.
+func (self *ApierV1) GetTPUserIds(attrs AttrGetTPUserIds, reply *[]string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if ids, err := self.StorDb.GetTpTableIds(attrs.TPid, utils.TBL_TP_USERS, utils.TPDistinctIds{"tenant", "user_name"}, nil, &attrs.Paginator); err != nil {
+		return utils.NewErrServerError(err)
+	} else if ids == nil {
+		return utils.ErrNotFound
+	} else {
+		*reply = ids
+	}
+	return nil
+}
+
+// Removes specific User on Tariff plan
+func (self *ApierV1) RemTPUser(attrs AttrGetTPUser, reply *string) error {
+	if missing := utils.MissingStructFields(&attrs, []string{"TPid", "UserId"}); len(missing) != 0 { //Params missing
+		return utils.NewErrMandatoryIeMissing(missing...)
+	}
+	if err := self.StorDb.RemTpData(utils.TBL_TP_USERS, attrs.TPid); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}

apier/v1/tutfsjson_local_test

@@ -0,0 +1,526 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package apier
+
+import (
+	"flag"
+	"fmt"
+	"net/rpc/jsonrpc"
+	"os"
+	"os/exec"
+	"path"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var fsjsonCfgPath string
+var fsjsonCfg *config.CGRConfig
+
+var waitFs = flag.Int("wait_fs", 500, "Number of miliseconds to wait for FreeSWITCH to start")
+
+func init() {
+	fsjsonCfgPath = path.Join(*dataDir, "tutorials", "fs_json", "cgrates", "etc", "cgrates", "cgrates.cfg")
+	fsjsonCfg, _ = config.NewCGRConfigFromFile(&fsjsonCfgPath)
+}
+
+// Remove here so they can be properly created by init script
+func TestFsJsonRemoveDirs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	for _, pathDir := range []string{cfg.CdreDefaultInstance.ExportDir, fsjsonCfg.HistoryDir} {
+		if err := os.RemoveAll(pathDir); err != nil {
+			t.Fatal("Error removing folder: ", pathDir, err)
+		}
+	}
+}
+
+// Empty tables before using them
+func TestFsJsonCreateTables(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if *storDbType != utils.MYSQL {
+		t.Fatal("Unsupported storDbType")
+	}
+	var mysql *engine.MySQLStorage
+	if d, err := engine.NewMySQLStorage(fsjsonCfg.StorDBHost, fsjsonCfg.StorDBPort, fsjsonCfg.StorDBName, fsjsonCfg.StorDBUser, fsjsonCfg.StorDBPass); err != nil {
+		t.Fatal("Error on opening database connection: ", err)
+	} else {
+		mysql = d.(*engine.MySQLStorage)
+	}
+	if err := mysql.CreateTablesFromScript(path.Join(*dataDir, "storage", *storDbType, engine.CREATE_CDRS_TABLES_SQL)); err != nil {
+		t.Fatal("Error on mysql creation: ", err.Error())
+		return // No point in going further
+	}
+	for _, tbl := range []string{utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_EXTRA} {
+		if _, err := mysql.Db.Query(fmt.Sprintf("SELECT 1 from %s", tbl)); err != nil {
+			t.Fatal(err.Error())
+		}
+	}
+}
+
+func TestFsJsonInitDataDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	ratingDb, err := engine.ConfigureRatingStorage(fsjsonCfg.RatingDBType, fsjsonCfg.RatingDBHost, fsjsonCfg.RatingDBPort, fsjsonCfg.RatingDBName, fsjsonCfg.RatingDBUser, fsjsonCfg.RatingDBPass, fsjsonCfg.DBDataEncoding)
+	if err != nil {
+		t.Fatal("Cannot connect to dataDb", err)
+	}
+	accountDb, err := engine.ConfigureAccountingStorage(fsjsonCfg.AccountDBType, fsjsonCfg.AccountDBHost, fsjsonCfg.AccountDBPort, fsjsonCfg.AccountDBName,
+		fsjsonCfg.AccountDBUser, fsjsonCfg.AccountDBPass, fsjsonCfg.DBDataEncoding)
+	if err != nil {
+		t.Fatal("Cannot connect to dataDb", err)
+	}
+	for _, db := range []engine.Storage{ratingDb, accountDb} {
+		if err := db.Flush(); err != nil {
+			t.Fatal("Cannot reset dataDb", err)
+		}
+	}
+}
+
+func TestFsJsonStartFs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	exec.Command("pkill", "freeswitch").Run() // Just to make sure another one is not running, bit brutal maybe we can fine tune it
+	go func() {
+		fs := exec.Command("/usr/share/cgrates/tutorials/fs_json/freeswitch/etc/init.d/freeswitch", "start")
+		out, _ := fs.CombinedOutput()
+		engine.Logger.Info(fmt.Sprintf("CgrEngine-TestFsJson: %s", out))
+	}()
+	time.Sleep(time.Duration(*waitFs) * time.Millisecond) // Give time to rater to fire up
+}
+
+// Finds cgr-engine executable and starts it with default configuration
+func TestFsJsonStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	exec.Command("pkill", "cgr-engine").Run() // Just to make sure another one is not running, bit brutal maybe we can fine tune it
+	go func() {
+		eng := exec.Command("/usr/share/cgrates/tutorials/fs_json/cgrates/etc/init.d/cgrates", "start")
+		out, _ := eng.CombinedOutput()
+		engine.Logger.Info(fmt.Sprintf("CgrEngine-TestFsJson: %s", out))
+	}()
+	time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time to rater to fire up
+}
+
+// Connect rpc client to rater
+func TestFsJsonRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	rater, err = jsonrpc.Dial("tcp", fsjsonCfg.RPCJSONListen)
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+// Make sure we start with fresh data
+func TestFsJsonEmptyCache(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var rcvStats *utils.CacheStats
+	expectedStats := &utils.CacheStats{Destinations: 0, RatingPlans: 0, RatingProfiles: 0, Actions: 0}
+	var args utils.AttrCacheStats
+	if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil {
+		t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())
+	} else if !reflect.DeepEqual(expectedStats, rcvStats) {
+		t.Errorf("Calling ApierV1.GetCacheStats expected: %v, received: %v", expectedStats, rcvStats)
+	}
+}
+
+func TestFsJsonLoadTariffPlans(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	reply := ""
+	// Simple test that command is executed without errors
+	attrs := &utils.AttrLoadTpFromFolder{FolderPath: path.Join(*dataDir, "tutorials", "fs_json", "cgrates", "tariffplans")}
+	if err := rater.Call("ApierV1.LoadTariffPlanFromFolder", attrs, &reply); err != nil {
+		t.Error("Got error on ApierV1.LoadTariffPlanFromFolder: ", err.Error())
+	} else if reply != "OK" {
+		t.Error("Calling ApierV1.LoadTariffPlanFromFolder got reply: ", reply)
+	}
+	time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time for scheduler to execute topups
+	var rcvStats *utils.CacheStats
+	expectedStats := &utils.CacheStats{Destinations: 3, RatingPlans: 3, RatingProfiles: 3, Actions: 5, SharedGroups: 1, RatingAliases: 1, AccountAliases: 1, DerivedChargers: 1}
+	var args utils.AttrCacheStats
+	if err := rater.Call("ApierV1.GetCacheStats", args, &rcvStats); err != nil {
+		t.Error("Got error on ApierV1.GetCacheStats: ", err.Error())
+	} else if !reflect.DeepEqual(expectedStats, rcvStats) {
+		t.Errorf("Calling ApierV1.GetCacheStats expected: %v, received: %v", expectedStats, rcvStats)
+	}
+}
+
+func TestFsJsonGetAccount1001(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var acnt *engine.Account
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001", Direction: "*out"}
+	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
+		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
+	}
+	if acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 10.0 {
+		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
+	}
+	if len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]) != 2 {
+		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]))
+	}
+	blncLst := acnt.BalanceMap[engine.CREDIT+attrs.Direction]
+	for _, blnc := range blncLst {
+		if len(blnc.SharedGroup) == 0 && blnc.Value != 5 {
+			t.Errorf("Unexpected value for general balance: %f", blnc.Value)
+		} else if blnc.SharedGroup == "SHARED_A" && blnc.Value != 5 {
+			t.Errorf("Unexpected value for shared balance: %f", blnc.Value)
+		}
+	}
+}
+
+func TestFsJsonGetAccount1002(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var acnt *engine.Account
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1002", Direction: "*out"}
+	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
+		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
+	}
+	if acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 10.0 {
+		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
+	}
+	if len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]) != 1 {
+		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]))
+	}
+	blnc := acnt.BalanceMap[engine.CREDIT+attrs.Direction][0]
+	if blnc.Value != 10 {
+		t.Errorf("Unexpected value for general balance: %f", blnc.Value)
+	}
+}
+
+func TestFsJsonGetAccount1003(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var acnt *engine.Account
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1003", Direction: "*out"}
+	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
+		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
+	}
+	if acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 10.0 {
+		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
+	}
+	if len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]) != 1 {
+		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]))
+	}
+	blnc := acnt.BalanceMap[engine.CREDIT+attrs.Direction][0]
+	if blnc.Value != 10 {
+		t.Errorf("Unexpected value for general balance: %f", blnc.Value)
+	}
+}
+
+func TestFsJsonGetAccount1004(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var acnt *engine.Account
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1004", Direction: "*out"}
+	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
+		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
+	}
+	if acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 10.0 {
+		t.Errorf("Calling ApierV1.GetBalance expected: 10.0, received: %f", acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
+	}
+	if len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]) != 1 {
+		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]))
+	}
+	blnc := acnt.BalanceMap[engine.CREDIT+attrs.Direction][0]
+	if blnc.Value != 10 {
+		t.Errorf("Unexpected value for general balance: %f", blnc.Value)
+	}
+}
+
+func TestFsJsonGetAccount1006(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var acnt *engine.Account
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1006", Direction: "*out"}
+	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err == nil {
+		t.Error("Got no error when querying unexisting balance")
+	}
+}
+
+func TestFsJsonGetAccount1007(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var acnt *engine.Account
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1007", Direction: "*out"}
+	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
+		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
+	}
+	if acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 0 {
+		t.Errorf("Calling ApierV1.GetBalance expected: 0, received: %f", acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
+	}
+	if len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]) != 1 {
+		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]))
+	}
+	blncLst := acnt.BalanceMap[engine.CREDIT+attrs.Direction]
+	for _, blnc := range blncLst {
+		if len(blnc.SharedGroup) == 0 && blnc.Value != 0 { // General balance
+			t.Errorf("Unexpected value for general balance: %f", blnc.Value)
+		} else if blnc.SharedGroup == "SHARED_A" && blnc.Value != 0 {
+			t.Errorf("Unexpected value for shared balance: %f", blnc.Value)
+		}
+	}
+}
+
+func TestMaxCallDuration(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	cd := engine.CallDescriptor{
+		Direction:   "*out",
+		Tenant:      "cgrates.org",
+		Category:    "call",
+		Subject:     "1001",
+		Account:     "1001",
+		Destination: "1002",
+		TimeStart:   time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC),
+		TimeEnd:     time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC).Add(fsjsonCfg.SMMaxCallDuration),
+	}
+	var remainingDurationFloat float64
+	if err := rater.Call("Responder.GetMaxSessionTime", cd, &remainingDurationFloat); err != nil {
+		t.Error(err)
+	} else {
+		remainingDuration := time.Duration(remainingDurationFloat)
+		if remainingDuration < time.Duration(90)*time.Minute {
+			t.Errorf("Expecting maxSessionTime around 1h30m, received as: %v", remainingDuration)
+		}
+	}
+	cd = engine.CallDescriptor{
+		Direction:   "*out",
+		Tenant:      "cgrates.org",
+		Category:    "call",
+		Subject:     "1002",
+		Account:     "1002",
+		Destination: "1001",
+		TimeStart:   time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC),
+		TimeEnd:     time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC).Add(fsjsonCfg.SMMaxCallDuration),
+	}
+	if err := rater.Call("Responder.GetMaxSessionTime", cd, &remainingDurationFloat); err != nil {
+		t.Error(err)
+	} else {
+		remainingDuration := time.Duration(remainingDurationFloat)
+		if remainingDuration < time.Duration(45)*time.Minute {
+			t.Errorf("Expecting maxSessionTime around 45m, received as: %v", remainingDuration)
+		}
+	}
+	cd = engine.CallDescriptor{
+		Direction:   "*out",
+		Tenant:      "cgrates.org",
+		Category:    "call",
+		Subject:     "1006",
+		Account:     "1006",
+		Destination: "1001",
+		TimeStart:   time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC),
+		TimeEnd:     time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC).Add(fsjsonCfg.SMMaxCallDuration),
+	}
+	if err := rater.Call("Responder.GetMaxSessionTime", cd, &remainingDurationFloat); err != nil {
+		t.Error(err)
+	} else {
+		remainingDuration := time.Duration(remainingDurationFloat)
+		if remainingDuration < time.Duration(45)*time.Minute {
+			t.Errorf("Expecting maxSessionTime around 45m, received as: %v", remainingDuration)
+		}
+	}
+	// 1007 should use the 1001 balance when doing maxSessionTime
+	cd = engine.CallDescriptor{
+		Direction:   "*out",
+		Tenant:      "cgrates.org",
+		Category:    "call",
+		Subject:     "1007",
+		Account:     "1007",
+		Destination: "1001",
+		TimeStart:   time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC),
+		TimeEnd:     time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC).Add(fsjsonCfg.SMMaxCallDuration),
+	}
+	if err := rater.Call("Responder.GetMaxSessionTime", cd, &remainingDurationFloat); err != nil {
+		t.Error(err)
+	} else {
+		remainingDuration := time.Duration(remainingDurationFloat)
+		if remainingDuration < time.Duration(20)*time.Minute {
+			t.Errorf("Expecting maxSessionTime around 20m, received as: %v", remainingDuration)
+		}
+	}
+}
+
+func TestMaxDebit1001(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	cc := &engine.CallCost{}
+	var acnt *engine.Account
+	cd := engine.CallDescriptor{
+		Direction:   "*out",
+		Tenant:      "cgrates.org",
+		Category:    "call",
+		Subject:     "1001",
+		Account:     "1001",
+		Destination: "1002",
+		TimeStart:   time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC),
+		TimeEnd:     time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC).Add(time.Duration(10) * time.Second),
+	}
+	if err := rater.Call("Responder.MaxDebit", cd, cc); err != nil {
+		t.Error(err.Error())
+	} else if cc.GetDuration() > time.Duration(1)*time.Minute {
+		t.Errorf("Unexpected call duration received: %v", cc.GetDuration())
+	}
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001", Direction: "*out"}
+	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
+		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
+	} else {
+		if len(acnt.BalanceMap["*monetary*out"]) != 2 {
+			t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap["*monetary*out"]))
+		}
+		blncLst := acnt.BalanceMap["*monetary*out"]
+		for _, blnc := range blncLst {
+			if blnc.SharedGroup == "SHARED_A" && blnc.Value != 5 {
+				t.Errorf("Unexpected value for shared balance: %f", blnc.Value)
+			} else if len(blnc.SharedGroup) == 0 && blnc.Value != 4.4 {
+				t.Errorf("Unexpected value for general balance: %f", blnc.Value)
+			}
+		}
+	}
+}
+
+func TestMaxDebit1007(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	cc := &engine.CallCost{}
+	var acnt *engine.Account
+	cd := engine.CallDescriptor{
+		Direction:   "*out",
+		Tenant:      "cgrates.org",
+		Category:    "call",
+		Subject:     "1007",
+		Account:     "1007",
+		Destination: "1002",
+		TimeStart:   time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC),
+		TimeEnd:     time.Date(2014, 3, 27, 10, 42, 26, 0, time.UTC).Add(time.Duration(10) * time.Second),
+	}
+	if err := rater.Call("Responder.MaxDebit", cd, cc); err != nil {
+		t.Error(err.Error())
+	} else if cc.GetDuration() > time.Duration(1)*time.Minute {
+		t.Errorf("Unexpected call duration received: %v", cc.GetDuration())
+	}
+	// Debit out of shared balance should reflect in the 1001 instead of 1007
+	attrs := &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1001", Direction: "*out"}
+	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
+		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
+	} else {
+		if len(acnt.BalanceMap["*monetary*out"]) != 2 {
+			t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap["*monetary*out"]))
+		}
+		blncLst := acnt.BalanceMap["*monetary*out"]
+		for _, blnc := range blncLst {
+			if blnc.SharedGroup == "SHARED_A" && blnc.Value != 4 {
+				t.Errorf("Unexpected value for shared balance: %f", blnc.Value)
+			} else if len(blnc.SharedGroup) == 0 && blnc.Value != 4.4 {
+				t.Errorf("Unexpected value for general balance: %f", blnc.Value)
+			}
+		}
+	}
+	// Make sure 1007 remains the same
+	attrs = &utils.AttrGetAccount{Tenant: "cgrates.org", Account: "1007", Direction: "*out"}
+	if err := rater.Call("ApierV1.GetAccount", attrs, &acnt); err != nil {
+		t.Error("Got error on ApierV1.GetAccount: ", err.Error())
+	}
+	if acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue() != 0 {
+		t.Errorf("Calling ApierV1.GetBalance expected: 0, received: %f", acnt.BalanceMap[engine.CREDIT+attrs.Direction].GetTotalValue())
+	}
+	if len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]) != 1 {
+		t.Errorf("Unexpected number of balances found: %d", len(acnt.BalanceMap[engine.CREDIT+attrs.Direction]))
+	}
+	blnc := acnt.BalanceMap[engine.CREDIT+attrs.Direction][0]
+	if len(blnc.SharedGroup) == 0 { // General balance
+		t.Errorf("Unexpected general balance: %f", blnc.Value)
+	} else if blnc.SharedGroup == "SHARED_A" && blnc.Value != 0 {
+		t.Errorf("Unexpected value for shared balance: %f", blnc.Value)
+	}
+}
+
+func TestDerivedChargers1001(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	attrs := &utils.AttrDerivedChargers{Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001"}
+	expectedDCs := utils.DerivedChargers([]*utils.DerivedCharger{
+		&utils.DerivedCharger{RunId: "fs_json_run", ReqTypeField: "^rated", DirectionField: "*default", TenantField: "*default", CategoryField: "*default",
+			AccountField: "*default", SubjectField: "^1002", DestinationField: "*default", SetupTimeField: "*default", AnswerTimeField: "*default", UsageField: "*default"},
+	})
+	var rcvRspDCs utils.DerivedChargers
+	if err := rater.Call("Responder.GetDerivedChargers", attrs, &rcvRspDCs); err != nil {
+		t.Error(err.Error())
+	} else if !reflect.DeepEqual(expectedDCs, rcvRspDCs) {
+		t.Errorf("Expected: %v: received: %v", expectedDCs, rcvRspDCs)
+	}
+	// Make sure that over Apier we get the same result as over Responder
+	var rcvApierDCs utils.DerivedChargers
+	if err := rater.Call("ApierV1.GetDerivedChargers", attrs, &rcvApierDCs); err != nil {
+		t.Error(err.Error())
+	} else if !reflect.DeepEqual(rcvRspDCs, rcvApierDCs) {
+		t.Errorf("Expected: %v: received: %v", rcvRspDCs, rcvApierDCs)
+	}
+}
+
+// Simply kill the engine after we are done with tests within this file
+func TestFsJsonStopEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	go func() {
+		eng := exec.Command("/usr/share/cgrates/tutorials/fs_json/cgrates/etc/init.d/cgrates", "stop")
+		out, _ := eng.CombinedOutput()
+		engine.Logger.Info(fmt.Sprintf("CgrEngine-TestFsJson: %s", out))
+	}()
+}
+
+func TestFsJsonStopFs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	go func() {
+		fs := exec.Command("/usr/share/cgrates/tutorials/fs_json/freeswitch/etc/init.d/freeswitch", "stop")
+		out, _ := fs.CombinedOutput()
+		engine.Logger.Info(fmt.Sprintf("CgrEngine-TestFsJson: %s", out))
+	}()
+}

apier/v2/apier.go

@@ -0,0 +1,258 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/cgrates/cgrates/apier/v1"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type ApierV2 struct {
+	v1.ApierV1
+}
+
+type AttrLoadRatingProfile struct {
+	TPid            string
+	RatingProfileId string
+}
+
+// Process dependencies and load a specific rating profile from storDb into dataDb.
+func (self *ApierV2) LoadRatingProfile(attrs AttrLoadRatingProfile, reply *string) error {
+	if len(attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	tpRpf := &utils.TPRatingProfile{TPid: attrs.TPid}
+	tpRpf.SetRatingProfilesId(attrs.RatingProfileId)
+	rpf := engine.APItoModelRatingProfile(tpRpf)
+	dbReader := engine.NewTpReader(self.RatingDb, self.AccountDb, self.StorDb, attrs.TPid, self.Config.DefaultTimezone, self.Config.LoadHistorySize)
+	if err := dbReader.LoadRatingProfilesFiltered(&rpf[0]); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	//Automatic cache of the newly inserted rating profile
+	var ratingProfile []string
+	if tpRpf.KeyId() != ":::" { // if has some filters
+		ratingProfile = []string{utils.RATING_PROFILE_PREFIX + tpRpf.KeyId()}
+	}
+	if err := self.RatingDb.CacheRatingPrefixValues(map[string][]string{utils.RATING_PROFILE_PREFIX: ratingProfile}); err != nil {
+		return err
+	}
+	*reply = v1.OK
+	return nil
+}
+
+type AttrLoadAccountActions struct {
+	TPid             string
+	AccountActionsId string
+}
+
+// Process dependencies and load a specific AccountActions profile from storDb into dataDb.
+func (self *ApierV2) LoadAccountActions(attrs AttrLoadAccountActions, reply *string) error {
+	if len(attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	dbReader := engine.NewTpReader(self.RatingDb, self.AccountDb, self.StorDb, attrs.TPid, self.Config.DefaultTimezone, self.Config.LoadHistorySize)
+	tpAa := &utils.TPAccountActions{TPid: attrs.TPid}
+	tpAa.SetAccountActionsId(attrs.AccountActionsId)
+	aa := engine.APItoModelAccountAction(tpAa)
+	if _, err := engine.Guardian.Guard(func() (interface{}, error) {
+		if err := dbReader.LoadAccountActionsFiltered(aa); err != nil {
+			return 0, err
+		}
+		return 0, nil
+	}, 0, attrs.AccountActionsId); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	// ToDo: Get the action keys loaded by dbReader so we reload only these in cache
+	// Need to do it before scheduler otherwise actions to run will be unknown
+	if err := self.RatingDb.CacheRatingPrefixes(utils.DERIVEDCHARGERS_PREFIX, utils.ACTION_PREFIX, utils.SHARED_GROUP_PREFIX); err != nil {
+		return err
+	}
+	if self.Sched != nil {
+		self.Sched.LoadActionPlans(self.RatingDb)
+		self.Sched.Restart()
+	}
+	*reply = v1.OK
+	return nil
+}
+
+type AttrLoadDerivedChargers struct {
+	TPid              string
+	DerivedChargersId string
+}
+
+// Load derived chargers from storDb into dataDb.
+func (self *ApierV2) LoadDerivedChargers(attrs AttrLoadDerivedChargers, reply *string) error {
+	if len(attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	tpDc := &utils.TPDerivedChargers{TPid: attrs.TPid}
+	tpDc.SetDerivedChargersId(attrs.DerivedChargersId)
+	dc := engine.APItoModelDerivedCharger(tpDc)
+	dbReader := engine.NewTpReader(self.RatingDb, self.AccountDb, self.StorDb, attrs.TPid, self.Config.DefaultTimezone, self.Config.LoadHistorySize)
+	if err := dbReader.LoadDerivedChargersFiltered(&dc[0], true); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	//Automatic cache of the newly inserted rating plan
+	var dcsChanged []string
+	if len(attrs.DerivedChargersId) != 0 {
+		dcsChanged = []string{utils.DERIVEDCHARGERS_PREFIX + attrs.DerivedChargersId}
+	}
+	if err := self.RatingDb.CacheRatingPrefixValues(map[string][]string{utils.DERIVEDCHARGERS_PREFIX: dcsChanged}); err != nil {
+		return err
+	}
+	*reply = v1.OK
+	return nil
+}
+
+func (self *ApierV2) LoadTariffPlanFromFolder(attrs utils.AttrLoadTpFromFolder, reply *engine.LoadInstance) error {
+	if len(attrs.FolderPath) == 0 {
+		return fmt.Errorf("%s:%s", utils.ErrMandatoryIeMissing.Error(), "FolderPath")
+	}
+	if fi, err := os.Stat(attrs.FolderPath); err != nil {
+		if strings.HasSuffix(err.Error(), "no such file or directory") {
+			return utils.ErrInvalidPath
+		}
+		return utils.NewErrServerError(err)
+	} else if !fi.IsDir() {
+		return utils.ErrInvalidPath
+	}
+	loader := engine.NewTpReader(self.RatingDb, self.AccountDb, engine.NewFileCSVStorage(utils.CSV_SEP,
+		path.Join(attrs.FolderPath, utils.DESTINATIONS_CSV),
+		path.Join(attrs.FolderPath, utils.TIMINGS_CSV),
+		path.Join(attrs.FolderPath, utils.RATES_CSV),
+		path.Join(attrs.FolderPath, utils.DESTINATION_RATES_CSV),
+		path.Join(attrs.FolderPath, utils.RATING_PLANS_CSV),
+		path.Join(attrs.FolderPath, utils.RATING_PROFILES_CSV),
+		path.Join(attrs.FolderPath, utils.SHARED_GROUPS_CSV),
+		path.Join(attrs.FolderPath, utils.LCRS_CSV),
+		path.Join(attrs.FolderPath, utils.ACTIONS_CSV),
+		path.Join(attrs.FolderPath, utils.ACTION_PLANS_CSV),
+		path.Join(attrs.FolderPath, utils.ACTION_TRIGGERS_CSV),
+		path.Join(attrs.FolderPath, utils.ACCOUNT_ACTIONS_CSV),
+		path.Join(attrs.FolderPath, utils.DERIVED_CHARGERS_CSV),
+		path.Join(attrs.FolderPath, utils.CDR_STATS_CSV),
+		path.Join(attrs.FolderPath, utils.USERS_CSV),
+		path.Join(attrs.FolderPath, utils.ALIASES_CSV),
+	), "", self.Config.DefaultTimezone, self.Config.LoadHistorySize)
+	if err := loader.LoadAll(); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if attrs.DryRun {
+		*reply = engine.LoadInstance{LoadId: utils.DRYRUN}
+		return nil // Mission complete, no errors
+	}
+
+	if attrs.Validate {
+		if !loader.IsValid() {
+			return errors.New("invalid data")
+		}
+	}
+
+	if err := loader.WriteToDatabase(attrs.FlushDb, false); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	// Make sure the items are in the cache
+	dstIds, _ := loader.GetLoadedIds(utils.DESTINATION_PREFIX)
+	dstKeys := make([]string, len(dstIds))
+	for idx, dId := range dstIds {
+		dstKeys[idx] = utils.DESTINATION_PREFIX + dId // Cache expects them as redis keys
+	}
+	rplIds, _ := loader.GetLoadedIds(utils.RATING_PLAN_PREFIX)
+	rpKeys := make([]string, len(rplIds))
+	for idx, rpId := range rplIds {
+		rpKeys[idx] = utils.RATING_PLAN_PREFIX + rpId
+	}
+	rpfIds, _ := loader.GetLoadedIds(utils.RATING_PROFILE_PREFIX)
+	rpfKeys := make([]string, len(rpfIds))
+	for idx, rpfId := range rpfIds {
+		rpfKeys[idx] = utils.RATING_PROFILE_PREFIX + rpfId
+	}
+	actIds, _ := loader.GetLoadedIds(utils.ACTION_PREFIX)
+	actKeys := make([]string, len(actIds))
+	for idx, actId := range actIds {
+		actKeys[idx] = utils.ACTION_PREFIX + actId
+	}
+	shgIds, _ := loader.GetLoadedIds(utils.SHARED_GROUP_PREFIX)
+	shgKeys := make([]string, len(shgIds))
+	for idx, shgId := range shgIds {
+		shgKeys[idx] = utils.SHARED_GROUP_PREFIX + shgId
+	}
+	aliases, _ := loader.GetLoadedIds(utils.ALIASES_PREFIX)
+	alsKeys := make([]string, len(aliases))
+	for idx, alias := range aliases {
+		alsKeys[idx] = utils.ALIASES_PREFIX + alias
+	}
+	lcrIds, _ := loader.GetLoadedIds(utils.LCR_PREFIX)
+	lcrKeys := make([]string, len(lcrIds))
+	for idx, lcrId := range lcrIds {
+		lcrKeys[idx] = utils.LCR_PREFIX + lcrId
+	}
+	dcs, _ := loader.GetLoadedIds(utils.DERIVEDCHARGERS_PREFIX)
+	dcsKeys := make([]string, len(dcs))
+	for idx, dc := range dcs {
+		dcsKeys[idx] = utils.DERIVEDCHARGERS_PREFIX + dc
+	}
+	aps, _ := loader.GetLoadedIds(utils.ACTION_TIMING_PREFIX)
+	engine.Logger.Info("ApierV1.LoadTariffPlanFromFolder, reloading cache.")
+
+	if err := self.RatingDb.CacheRatingPrefixValues(map[string][]string{
+		utils.DESTINATION_PREFIX:     dstKeys,
+		utils.RATING_PLAN_PREFIX:     rpKeys,
+		utils.RATING_PROFILE_PREFIX:  rpfKeys,
+		utils.LCR_PREFIX:             lcrKeys,
+		utils.DERIVEDCHARGERS_PREFIX: dcsKeys,
+		utils.ACTION_PREFIX:          actKeys,
+		utils.SHARED_GROUP_PREFIX:    shgKeys,
+	}); err != nil {
+		return err
+	}
+	if err := self.AccountDb.CacheAccountingPrefixValues(map[string][]string{
+		utils.ALIASES_PREFIX: alsKeys,
+	}); err != nil {
+		return err
+	}
+	if len(aps) != 0 && self.Sched != nil {
+		engine.Logger.Info("ApierV1.LoadTariffPlanFromFolder, reloading scheduler.")
+		self.Sched.LoadActionPlans(self.RatingDb)
+		self.Sched.Restart()
+	}
+	cstKeys, _ := loader.GetLoadedIds(utils.CDR_STATS_PREFIX)
+	if len(cstKeys) != 0 && self.CdrStatsSrv != nil {
+		if err := self.CdrStatsSrv.ReloadQueues(cstKeys, nil); err != nil {
+			return err
+		}
+	}
+
+	userKeys, _ := loader.GetLoadedIds(utils.USERS_PREFIX)
+	if len(userKeys) != 0 && self.Users != nil {
+		var r string
+		if err := self.Users.ReloadUsers("", &r); err != nil {
+			return err
+		}
+	}
+	*reply = *loader.GetLoadInstance()
+	return nil
+}

apier/v2/cdre.go

@@ -0,0 +1,136 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/cgrates/cgrates/cdre"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Export Cdrs to file
+func (self *ApierV2) ExportCdrsToFile(attr utils.AttrExportCdrsToFile, reply *utils.ExportedFileCdrs) error {
+	var err error
+	cdreReloadStruct := <-self.Config.ConfigReloads[utils.CDRE]                  // Read the content of the channel, locking it
+	defer func() { self.Config.ConfigReloads[utils.CDRE] <- cdreReloadStruct }() // Unlock reloads at exit
+	exportTemplate := self.Config.CdreProfiles[utils.META_DEFAULT]
+	if attr.ExportTemplate != nil && len(*attr.ExportTemplate) != 0 { // Export template prefered, use it
+		var hasIt bool
+		if exportTemplate, hasIt = self.Config.CdreProfiles[*attr.ExportTemplate]; !hasIt {
+			return fmt.Errorf("%s:ExportTemplate", utils.ErrNotFound)
+		}
+	}
+	cdrFormat := exportTemplate.CdrFormat
+	if attr.CdrFormat != nil && len(*attr.CdrFormat) != 0 {
+		cdrFormat = strings.ToLower(*attr.CdrFormat)
+	}
+	if !utils.IsSliceMember(utils.CdreCdrFormats, cdrFormat) {
+		return utils.NewErrMandatoryIeMissing("CdrFormat")
+	}
+	fieldSep := exportTemplate.FieldSeparator
+	if attr.FieldSeparator != nil && len(*attr.FieldSeparator) != 0 {
+		fieldSep, _ = utf8.DecodeRuneInString(*attr.FieldSeparator)
+		if fieldSep == utf8.RuneError {
+			return fmt.Errorf("%s:FieldSeparator:%s", utils.ErrServerError, "Invalid")
+		}
+	}
+	exportDir := exportTemplate.ExportDir
+	if attr.ExportDir != nil && len(*attr.ExportDir) != 0 {
+		exportDir = *attr.ExportDir
+	}
+	exportId := strconv.FormatInt(time.Now().Unix(), 10)
+	if attr.ExportId != nil && len(*attr.ExportId) != 0 {
+		exportId = *attr.ExportId
+	}
+	fileName := fmt.Sprintf("cdre_%s.%s", exportId, cdrFormat)
+	if attr.ExportFileName != nil && len(*attr.ExportFileName) != 0 {
+		fileName = *attr.ExportFileName
+	}
+	filePath := path.Join(exportDir, fileName)
+	if cdrFormat == utils.DRYRUN {
+		filePath = utils.DRYRUN
+	}
+	dataUsageMultiplyFactor := exportTemplate.DataUsageMultiplyFactor
+	if attr.DataUsageMultiplyFactor != nil && *attr.DataUsageMultiplyFactor != 0.0 {
+		dataUsageMultiplyFactor = *attr.DataUsageMultiplyFactor
+	}
+	smsUsageMultiplyFactor := exportTemplate.SmsUsageMultiplyFactor
+	if attr.SmsUsageMultiplyFactor != nil && *attr.SmsUsageMultiplyFactor != 0.0 {
+		smsUsageMultiplyFactor = *attr.SmsUsageMultiplyFactor
+	}
+	genericUsageMultiplyFactor := exportTemplate.GenericUsageMultiplyFactor
+	if attr.GenericUsageMultiplyFactor != nil && *attr.GenericUsageMultiplyFactor != 0.0 {
+		genericUsageMultiplyFactor = *attr.GenericUsageMultiplyFactor
+	}
+	costMultiplyFactor := exportTemplate.CostMultiplyFactor
+	if attr.CostMultiplyFactor != nil && *attr.CostMultiplyFactor != 0.0 {
+		costMultiplyFactor = *attr.CostMultiplyFactor
+	}
+	costShiftDigits := exportTemplate.CostShiftDigits
+	if attr.CostShiftDigits != nil {
+		costShiftDigits = *attr.CostShiftDigits
+	}
+	roundingDecimals := exportTemplate.CostRoundingDecimals
+	if attr.RoundDecimals != nil {
+		roundingDecimals = *attr.RoundDecimals
+	}
+	maskDestId := exportTemplate.MaskDestId
+	if attr.MaskDestinationId != nil && len(*attr.MaskDestinationId) != 0 {
+		maskDestId = *attr.MaskDestinationId
+	}
+	maskLen := exportTemplate.MaskLength
+	if attr.MaskLength != nil {
+		maskLen = *attr.MaskLength
+	}
+	cdrsFltr, err := attr.RpcCdrsFilter.AsCdrsFilter(self.Config.DefaultTimezone)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	cdrs, _, err := self.CdrDb.GetStoredCdrs(cdrsFltr)
+	if err != nil {
+		return err
+	} else if len(cdrs) == 0 {
+		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
+		return nil
+	}
+	cdrexp, err := cdre.NewCdrExporter(cdrs, self.CdrDb, exportTemplate, cdrFormat, fieldSep, exportId, dataUsageMultiplyFactor, smsUsageMultiplyFactor, genericUsageMultiplyFactor,
+		costMultiplyFactor, costShiftDigits, roundingDecimals, self.Config.RoundingDecimals, maskDestId, maskLen, self.Config.HttpSkipTlsVerify, self.Config.DefaultTimezone)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if cdrexp.TotalExportedCdrs() == 0 {
+		*reply = utils.ExportedFileCdrs{ExportedFilePath: ""}
+		return nil
+	}
+	if err := cdrexp.WriteToFile(filePath); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = utils.ExportedFileCdrs{ExportedFilePath: filePath, TotalRecords: len(cdrs), TotalCost: cdrexp.TotalCost(), FirstOrderId: cdrexp.FirstOrderId(), LastOrderId: cdrexp.LastOrderId()}
+	if !attr.SuppressCgrIds {
+		reply.ExportedCgrIds = cdrexp.PositiveExports()
+		reply.UnexportedCgrIds = cdrexp.NegativeExports()
+	}
+	return nil
+}

apier/v2/cdrs.go

@@ -0,0 +1,63 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"github.com/cgrates/cgrates/apier/v1"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Retrieves CDRs based on the filters
+func (apier *ApierV2) GetCdrs(attrs utils.RpcCdrsFilter, reply *[]*engine.ExternalCdr) error {
+	cdrsFltr, err := attrs.AsCdrsFilter(apier.Config.DefaultTimezone)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if cdrs, _, err := apier.CdrDb.GetStoredCdrs(cdrsFltr); err != nil {
+		return utils.NewErrServerError(err)
+	} else if len(cdrs) == 0 {
+		*reply = make([]*engine.ExternalCdr, 0)
+	} else {
+		for _, cdr := range cdrs {
+			*reply = append(*reply, cdr.AsExternalCdr())
+		}
+	}
+	return nil
+}
+
+func (apier *ApierV2) CountCdrs(attrs utils.RpcCdrsFilter, reply *int64) error {
+	cdrsFltr, err := attrs.AsCdrsFilter(apier.Config.DefaultTimezone)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	cdrsFltr.Count = true
+	if _, count, err := apier.CdrDb.GetStoredCdrs(cdrsFltr); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = count
+	}
+	return nil
+}
+
+// Receive CDRs via RPC methods, not included with APIer because it has way less dependencies and can be standalone
+type CdrsV2 struct {
+	v1.CdrsV1
+}

apier/v2/cdrs_mysql_local_test.go

@@ -0,0 +1,250 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"flag"
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"path"
+	"testing"
+	"time"
+)
+
+var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args
+var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here")
+var waitRater = flag.Int("wait_rater", 500, "Number of miliseconds to wait for rater to start and cache")
+
+var cdrsCfgPath string
+var cdrsCfg *config.CGRConfig
+var cdrsRpc *rpc.Client
+
+func TestV2CdrsMysqlInitConfig(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrsCfgPath = path.Join(*dataDir, "conf", "samples", "cdrsv2mysql")
+	if cdrsCfg, err = config.NewCGRConfigFromFolder(cdrsCfgPath); err != nil {
+		t.Fatal("Got config error: ", err.Error())
+	}
+}
+
+func TestV2CdrsMysqlInitDataDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitDataDb(cdrsCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// InitDb so we can rely on count
+func TestV2CdrsMysqlInitCdrDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitStorDb(cdrsCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestV2CdrsMysqlInjectUnratedCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var mysqlDb *engine.MySQLStorage
+	if d, err := engine.NewMySQLStorage(cdrsCfg.StorDBHost, cdrsCfg.StorDBPort, cdrsCfg.StorDBName, cdrsCfg.StorDBUser, cdrsCfg.StorDBPass,
+		cdrsCfg.StorDBMaxOpenConns, cdrsCfg.StorDBMaxIdleConns); err != nil {
+		t.Error("Error on opening database connection: ", err)
+		return
+	} else {
+		mysqlDb = d.(*engine.MySQLStorage)
+	}
+	strCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()),
+		TOR: utils.VOICE, AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: utils.META_RATED,
+		Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+		SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC),
+		Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
+		MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201}
+	if err := mysqlDb.SetCdr(strCdr1); err != nil {
+		t.Error(err.Error())
+	}
+}
+
+func TestV2CdrsMysqlStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if _, err := engine.StopStartEngine(cdrsCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestV2CdrsMysqlRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrsRpc, err = jsonrpc.Dial("tcp", cdrsCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+// Insert some CDRs
+func TestV2CdrsMysqlProcessCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans", Rated: true,
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("abcdeftg", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("aererfddf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+	}
+	for _, cdr := range cdrs {
+		if err := cdrsRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil {
+			t.Error("Unexpected error: ", err.Error())
+		} else if reply != utils.OK {
+			t.Error("Unexpected reply received: ", reply)
+		}
+	}
+}
+
+func TestV2CdrsMysqlGetCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply []*engine.ExternalCdr
+	req := utils.RpcCdrsFilter{}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 4 {
+		t.Error("Unexpected number of CDRs returned: ", len(reply))
+	}
+	// CDRs with errors
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0), MaxCost: utils.Float64Pointer(0.0)}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 2 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// CDRs Rated
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 3 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// CDRs non rated OR SkipRated
+	req = utils.RpcCdrsFilter{MaxCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 1 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// Skip Errors
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(0.0), MaxCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 2 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+}
+
+func TestV2CdrsMysqlCountCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply int64
+	req := utils.AttrGetCdrs{}
+	if err := cdrsRpc.Call("ApierV2.CountCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if reply != 4 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+}
+
+// Test Prepaid CDRs without previous costs being calculated
+func TestV2CdrsMysqlProcessPrepaidCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans", Rated: true,
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("abcdeftg2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("aererfddf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+	}
+	tStart := time.Now()
+	for _, cdr := range cdrs {
+		if err := cdrsRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil {
+			t.Error("Unexpected error: ", err.Error())
+		} else if reply != utils.OK {
+			t.Error("Unexpected reply received: ", reply)
+		}
+	}
+	if processDur := time.Now().Sub(tStart); processDur > 1*time.Second {
+		t.Error("Unexpected processing time", processDur)
+	}
+}
+
+func TestV2CdrsMysqlKillEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.KillEngine(*waitRater); err != nil {
+		t.Error(err)
+	}
+}

apier/v2/cdrs_psql_local_test.go

@@ -0,0 +1,248 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"os/exec"
+	"path"
+	"testing"
+	"time"
+)
+
+var cdrsPsqlCfgPath string
+var cdrsPsqlCfg *config.CGRConfig
+var cdrsPsqlRpc *rpc.Client
+var cmdEngineCdrPsql *exec.Cmd
+
+func TestV2CdrsPsqlInitConfig(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrsPsqlCfgPath = path.Join(*dataDir, "conf", "samples", "cdrsv2psql")
+	if cdrsPsqlCfg, err = config.NewCGRConfigFromFolder(cdrsPsqlCfgPath); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestV2CdrsPsqlInitDataDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitDataDb(cdrsPsqlCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// InitDb so we can rely on count
+func TestV2CdrsPsqlInitCdrDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitStorDb(cdrsPsqlCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestV2CdrsPsqlInjectUnratedCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var psqlDb *engine.PostgresStorage
+	if d, err := engine.NewPostgresStorage(cdrsPsqlCfg.StorDBHost, cdrsPsqlCfg.StorDBPort, cdrsPsqlCfg.StorDBName, cdrsPsqlCfg.StorDBUser, cdrsPsqlCfg.StorDBPass,
+		cdrsPsqlCfg.StorDBMaxOpenConns, cdrsPsqlCfg.StorDBMaxIdleConns); err != nil {
+		t.Error("Error on opening database connection: ", err)
+		return
+	} else {
+		psqlDb = d.(*engine.PostgresStorage)
+	}
+	strCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("bbb1", time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC).String()),
+		TOR: utils.VOICE, AccId: "bbb1", CdrHost: "192.168.1.1", CdrSource: "UNKNOWN", ReqType: utils.META_RATED,
+		Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+		SetupTime: time.Date(2013, 12, 7, 8, 42, 24, 0, time.UTC), AnswerTime: time.Date(2013, 12, 7, 8, 42, 26, 0, time.UTC),
+		Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
+		MediationRunId: utils.DEFAULT_RUNID, Cost: 1.201}
+	if err := psqlDb.SetCdr(strCdr1); err != nil {
+		t.Error(err.Error())
+	}
+}
+
+func TestV2CdrsPsqlStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	if cmdEngineCdrPsql, err = engine.StartEngine(cdrsPsqlCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestV2CdrsPsqlPsqlRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	cdrsPsqlRpc, err = jsonrpc.Dial("tcp", cdrsPsqlCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+// Insert some CDRs
+func TestV2CdrsPsqlProcessCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans", Rated: true,
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("abcdeftg", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("aererfddf", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+	}
+	for _, cdr := range cdrs {
+		if err := cdrsPsqlRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil {
+			t.Error("Unexpected error: ", err.Error())
+		} else if reply != utils.OK {
+			t.Error("Unexpected reply received: ", reply)
+		}
+	}
+}
+
+func TestV2CdrsPsqlGetCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply []*engine.ExternalCdr
+	req := utils.RpcCdrsFilter{}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 4 {
+		t.Error("Unexpected number of CDRs returned: ", len(reply))
+	}
+	// CDRs with errors
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0), MaxCost: utils.Float64Pointer(0.0)}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 2 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// CDRs Rated
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 3 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// CDRs non rated OR SkipRated
+	req = utils.RpcCdrsFilter{MaxCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 1 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+	// Skip Errors
+	req = utils.RpcCdrsFilter{MinCost: utils.Float64Pointer(0.0), MaxCost: utils.Float64Pointer(-1.0)}
+	if err := cdrsPsqlRpc.Call("ApierV2.GetCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if len(reply) != 2 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+}
+
+func TestV2CdrsPsqlCountCdrs(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply int64
+	req := utils.AttrGetCdrs{}
+	if err := cdrsPsqlRpc.Call("ApierV2.CountCdrs", req, &reply); err != nil {
+		t.Error("Unexpected error: ", err.Error())
+	} else if reply != 4 {
+		t.Error("Unexpected number of CDRs returned: ", reply)
+	}
+}
+
+// Test Prepaid CDRs without previous costs being calculated
+func TestV2CdrsPsqlProcessPrepaidCdr(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var reply string
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans", Rated: true,
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("abcdeftg2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1002", Subject: "1002", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+		&engine.StoredCdr{CgrId: utils.Sha1("aererfddf2", time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC).String()), OrderId: 123, TOR: utils.VOICE, AccId: "dsafdsaf",
+			CdrHost: "192.168.1.1", CdrSource: "test", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org", Category: "call", Account: "1003", Subject: "1003", Destination: "1002",
+			SetupTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC), MediationRunId: utils.DEFAULT_RUNID,
+			Usage: time.Duration(10) * time.Second, ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"}, Cost: 1.01,
+			RatedAccount: "dan", RatedSubject: "dans",
+		},
+	}
+	tStart := time.Now()
+	for _, cdr := range cdrs {
+		if err := cdrsPsqlRpc.Call("CdrsV2.ProcessCdr", cdr, &reply); err != nil {
+			t.Error("Unexpected error: ", err.Error())
+		} else if reply != utils.OK {
+			t.Error("Unexpected reply received: ", reply)
+		}
+	}
+	if processDur := time.Now().Sub(tStart); processDur > 1*time.Second {
+		t.Error("Unexpected processing time", processDur)
+	}
+}
+
+func TestV2CdrsPsqlKillEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.KillEngine(*waitRater); err != nil {
+		t.Error(err)
+	}
+}

apier/v2/tp.go

@@ -0,0 +1,99 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package v2
+
+import (
+	"encoding/base64"
+
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+type AttrRemTp struct {
+	TPid string
+}
+
+func (self *ApierV2) RemTP(attrs AttrRemTp, reply *string) error {
+	if len(attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	if err := self.StorDb.RemTpData("", attrs.TPid); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*reply = "OK"
+	}
+	return nil
+}
+
+func (self *ApierV2) ExportTPToFolder(attrs utils.AttrDirExportTP, exported *utils.ExportedTPStats) error {
+	if len(*attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	dir := self.Config.TpExportPath
+	if attrs.ExportPath != nil {
+		dir = *attrs.ExportPath
+	}
+	fileFormat := utils.CSV
+	if attrs.FileFormat != nil {
+		fileFormat = *attrs.FileFormat
+	}
+	sep := ","
+	if attrs.FieldSeparator != nil {
+		sep = *attrs.FieldSeparator
+	}
+	compress := false
+	if attrs.Compress != nil {
+		compress = *attrs.Compress
+	}
+	tpExporter, err := engine.NewTPExporter(self.StorDb, *attrs.TPid, dir, fileFormat, sep, compress)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if err := tpExporter.Run(); err != nil {
+		return utils.NewErrServerError(err)
+	} else {
+		*exported = *tpExporter.ExportStats()
+	}
+
+	return nil
+}
+
+func (self *ApierV2) ExportTPToZipString(attrs utils.AttrDirExportTP, reply *string) error {
+	if len(*attrs.TPid) == 0 {
+		return utils.NewErrMandatoryIeMissing("TPid")
+	}
+	dir := ""
+	fileFormat := utils.CSV
+	if attrs.FileFormat != nil {
+		fileFormat = *attrs.FileFormat
+	}
+	sep := ","
+	if attrs.FieldSeparator != nil {
+		sep = *attrs.FieldSeparator
+	}
+	tpExporter, err := engine.NewTPExporter(self.StorDb, *attrs.TPid, dir, fileFormat, sep, true)
+	if err != nil {
+		return utils.NewErrServerError(err)
+	}
+	if err := tpExporter.Run(); err != nil {
+		return utils.NewErrServerError(err)
+	}
+	*reply = base64.StdEncoding.EncodeToString(tpExporter.GetCacheBuffer().Bytes())
+	return nil
+}

build.sh

@@ -1,12 +1,14 @@
 #! /usr/bin/env sh
 
-go install github.com/cgrates/cgrates/cmd/cgr-rater
+echo "Building CGRateS..."
+
+go install github.com/cgrates/cgrates/cmd/cgr-engine
 cr=$?
 go install github.com/cgrates/cgrates/cmd/cgr-loader
 cl=$?
 go install github.com/cgrates/cgrates/cmd/cgr-console
 cc=$?
+go install github.com/cgrates/cgrates/cmd/cgr-tester
+ct=$?
 
-exit $cr || $cl || $cc
-
-
+exit $cr || $cl || $cc || $ct

cache2go/cache.go

@@ -2,116 +2,176 @@
 package cache2go
 
 import (
-	"errors"
 	"sync"
 	"time"
 )
 
-type expiringCacheEntry interface {
-	XCache(key string, expire time.Duration, value expiringCacheEntry)
-	timer() *time.Timer
-	KeepAlive()
+const (
+	PREFIX_LEN   = 4
+	KIND_ADD     = "ADD"
+	KIND_ADP     = "ADP"
+	KIND_REM     = "REM"
+	KIND_PRF     = "PRF"
+	DOUBLE_CACHE = true
+)
+
+type timestampedValue struct {
+	timestamp time.Time
+	value     interface{}
 }
 
-// Structure that must be embeded in the objectst that must be cached with expiration.
-// If the expiration is not needed this can be ignored
-type XEntry struct {
-	sync.Mutex
-	key            string
-	keepAlive      bool
-	expireDuration time.Duration
-	t              *time.Timer
+func (tsv timestampedValue) Value() interface{} {
+	return tsv.value
+}
+
+type transactionItem struct {
+	key   string
+	value interface{}
+	kind  string
+}
+
+func init() {
+	if DOUBLE_CACHE {
+		cache = newDoubleStore()
+	} else {
+		cache = newSimpleStore()
+	}
 }
 
 var (
-	xcache = make(map[string]expiringCacheEntry)
-	xMux   sync.RWMutex
-	cache  = make(map[string]interface{})
-	mux    sync.RWMutex
+	mux   sync.RWMutex
+	cache cacheStore
+	// transaction stuff
+	transactionBuffer []transactionItem
+	transactionMux    sync.Mutex
+	transactionON     = false
+	transactionLock   = false
 )
 
-// The main function to cache with expiration
-func (xe *XEntry) XCache(key string, expire time.Duration, value expiringCacheEntry) {
-	xe.keepAlive = true
-	xe.key = key
-	xe.expireDuration = expire
-	xMux.Lock()
-	xcache[key] = value
-	xMux.Unlock()
-	go xe.expire()
+func BeginTransaction() {
+	transactionMux.Lock()
+	transactionLock = true
+	transactionON = true
 }
 
-// The internal mechanism for expiartion
-func (xe *XEntry) expire() {
-	for xe.keepAlive {
-		xe.Lock()
-		xe.keepAlive = false
-		xe.Unlock()
-		xe.t = time.NewTimer(xe.expireDuration)
-		<-xe.t.C
-		if !xe.keepAlive {
-			xMux.Lock()
-			delete(xcache, xe.key)
-			xMux.Unlock()
+func RollbackTransaction() {
+	transactionBuffer = nil
+	transactionLock = false
+	transactionON = false
+	transactionMux.Unlock()
+}
+
+func CommitTransaction() {
+	transactionON = false
+	// apply all transactioned items
+	mux.Lock()
+	for _, item := range transactionBuffer {
+		switch item.kind {
+		case KIND_REM:
+			RemKey(item.key)
+		case KIND_PRF:
+			RemPrefixKey(item.key)
+		case KIND_ADD:
+			Cache(item.key, item.value)
+		case KIND_ADP:
+			CachePush(item.key, item.value)
 		}
 	}
-}
-
-// Getter for the timer
-func (xe *XEntry) timer() *time.Timer {
-	return xe.t
-}
-
-// Mark entry to be kept another expirationDuration period
-func (xe *XEntry) KeepAlive() {
-	xe.Lock()
-	defer xe.Unlock()
-	xe.keepAlive = true
-}
-
-// Get an entry from the expiration cache and mark it for keeping alive
-func GetXCached(key string) (ece expiringCacheEntry, err error) {
-	xMux.RLock()
-	defer xMux.RUnlock()
-	if r, ok := xcache[key]; ok {
-		r.KeepAlive()
-		return r, nil
-	}
-	return nil, errors.New("not found")
+	mux.Unlock()
+	transactionBuffer = nil
+	transactionLock = false
+	transactionMux.Unlock()
 }
 
 // The function to be used to cache a key/value pair when expiration is not needed
 func Cache(key string, value interface{}) {
-	mux.Lock()
-	defer mux.Unlock()
-	cache[key] = value
+	if !transactionLock {
+		mux.Lock()
+		defer mux.Unlock()
+	}
+	if !transactionON {
+		cache.Put(key, value)
+		//fmt.Println("ADD: ", key)
+	} else {
+		transactionBuffer = append(transactionBuffer, transactionItem{key: key, value: value, kind: KIND_ADD})
+	}
+}
+
+// Appends to an existing slice in the cache key
+func CachePush(key string, value interface{}) {
+	if !transactionLock {
+		mux.Lock()
+		defer mux.Unlock()
+	}
+	if !transactionON {
+		cache.Append(key, value)
+	} else {
+		transactionBuffer = append(transactionBuffer, transactionItem{key: key, value: value, kind: KIND_ADP})
+	}
 }
 
 // The function to extract a value for a key that never expire
-func GetCached(key string) (v interface{}, err error) {
+func Get(key string) (v interface{}, err error) {
 	mux.RLock()
 	defer mux.RUnlock()
-	if r, ok := cache[key]; ok {
-		return r, nil
-	}
-	return nil, errors.New("not found")
+	return cache.Get(key)
 }
 
-// Delete all keys from expiraton cache
-func XFlush() {
-	xMux.Lock()
-	defer xMux.Unlock()
-	for _, v := range xcache {
-		if v.timer() != nil {
-			v.timer().Stop()
-		}
+func GetKeyAge(key string) (time.Duration, error) {
+	mux.RLock()
+	defer mux.RUnlock()
+	return cache.GetAge(key)
+}
+
+func RemKey(key string) {
+	if !transactionLock {
+		mux.Lock()
+		defer mux.Unlock()
+	}
+	if !transactionON {
+		cache.Delete(key)
+	} else {
+		transactionBuffer = append(transactionBuffer, transactionItem{key: key, kind: KIND_REM})
+	}
+}
+
+func RemPrefixKey(prefix string) {
+	if !transactionLock {
+		mux.Lock()
+		defer mux.Unlock()
+	}
+	if !transactionON {
+		cache.DeletePrefix(prefix)
+	} else {
+		transactionBuffer = append(transactionBuffer, transactionItem{key: prefix, kind: KIND_PRF})
 	}
-	xcache = make(map[string]expiringCacheEntry)
 }
 
 // Delete all keys from cache
 func Flush() {
 	mux.Lock()
 	defer mux.Unlock()
-	cache = make(map[string]interface{})
+	if DOUBLE_CACHE {
+		cache = newDoubleStore()
+	} else {
+		cache = newSimpleStore()
+	}
+}
+
+func CountEntries(prefix string) (result int) {
+	mux.RLock()
+	defer mux.RUnlock()
+	return cache.CountEntriesForPrefix(prefix)
+}
+
+func GetAllEntries(prefix string) (map[string]timestampedValue, error) {
+	mux.RLock()
+	defer mux.RUnlock()
+	return cache.GetAllForPrefix(prefix)
+}
+
+func GetEntriesKeys(prefix string) (keys []string) {
+	mux.RLock()
+	defer mux.RUnlock()
+	return cache.GetKeysForPrefix(prefix)
 }

cache2go/cache_test.go

@@ -1,75 +1,106 @@
 package cache2go
 
-import (
-	"testing"
-	"time"
-)
+import "testing"
 
-type myStruct struct {
-	XEntry
-	data string
-}
-
-func TestCache(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 1*time.Second, a)
-	b, err := GetXCached("mama")
-	if err != nil || b == nil || b != a {
-		t.Error("Error retriving data from cache", err)
+func TestRemKey(t *testing.T) {
+	Cache("t11_mm", "test")
+	if t1, err := Get("t11_mm"); err != nil || t1 != "test" {
+		t.Error("Error setting cache: ", err, t1)
+	}
+	RemKey("t11_mm")
+	if t1, err := Get("t11_mm"); err == nil || t1 == "test" {
+		t.Error("Error removing cached key")
 	}
 }
 
-func TestCacheExpire(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 1*time.Second, a)
-	b, err := GetXCached("mama")
-	if err != nil || b == nil || b.(*myStruct).data != "mama are mere" {
-		t.Error("Error retriving data from cache", err)
+func TestTransaction(t *testing.T) {
+	BeginTransaction()
+	Cache("t11_mm", "test")
+	if t1, err := Get("t11_mm"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
 	}
-	time.Sleep(1001 * time.Millisecond)
-	b, err = GetXCached("mama")
-	if err == nil || b != nil {
-		t.Error("Error expiring data")
+	Cache("t12_mm", "test")
+	RemKey("t11_mm")
+	CommitTransaction()
+	if t1, err := Get("t12_mm"); err != nil || t1 != "test" {
+		t.Error("Error commiting transaction")
+	}
+	if t1, err := Get("t11_mm"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
 	}
 }
 
-func TestCacheKeepAlive(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 1*time.Second, a)
-	b, err := GetXCached("mama")
-	if err != nil || b == nil || b.(*myStruct).data != "mama are mere" {
-		t.Error("Error retriving data from cache", err)
+func TestTransactionRem(t *testing.T) {
+	BeginTransaction()
+	Cache("t21_mm", "test")
+	Cache("t21_nn", "test")
+	RemPrefixKey("t21_")
+	CommitTransaction()
+	if t1, err := Get("t21_mm"); err == nil || t1 == "test" {
+		t.Error("Error commiting transaction")
 	}
-	time.Sleep(500 * time.Millisecond)
-	b.KeepAlive()
-	time.Sleep(501 * time.Millisecond)
-	if err != nil {
-		t.Error("Error keeping cached data alive", err)
-	}
-	time.Sleep(1000 * time.Millisecond)
-	b, err = GetXCached("mama")
-	if err == nil || b != nil {
-		t.Error("Error expiring data")
+	if t1, err := Get("t21_nn"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
 	}
 }
 
-func TestFlush(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 10*time.Second, a)
-	time.Sleep(1000 * time.Millisecond)
-	XFlush()
-	b, err := GetXCached("mama")
-	if err == nil || b != nil {
-		t.Error("Error expiring data")
+func TestTransactionRollback(t *testing.T) {
+	BeginTransaction()
+	Cache("t31_mm", "test")
+	if t1, err := Get("t31_mm"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
+	}
+	Cache("t32_mm", "test")
+	RollbackTransaction()
+	if t1, err := Get("t32_mm"); err == nil || t1 == "test" {
+		t.Error("Error commiting transaction")
+	}
+	if t1, err := Get("t31_mm"); err == nil || t1 == "test" {
+		t.Error("Error in transaction cache")
 	}
 }
 
-func TestFlushNoTimout(t *testing.T) {
-	a := &myStruct{data: "mama are mere"}
-	a.XCache("mama", 10*time.Second, a)
-	XFlush()
-	b, err := GetXCached("mama")
-	if err == nil || b != nil {
-		t.Error("Error expiring data")
+func TestTransactionRemBefore(t *testing.T) {
+	BeginTransaction()
+	RemPrefixKey("t41_")
+	Cache("t41_mm", "test")
+	Cache("t41_nn", "test")
+	CommitTransaction()
+	if t1, err := Get("t41_mm"); err != nil || t1 != "test" {
+		t.Error("Error commiting transaction")
+	}
+	if t1, err := Get("t41_nn"); err != nil || t1 != "test" {
+		t.Error("Error in transaction cache")
+	}
+}
+
+func TestRemPrefixKey(t *testing.T) {
+	Cache("xxx_t1", "test")
+	Cache("yyy_t1", "test")
+	RemPrefixKey("xxx_")
+	_, errX := Get("xxx_t1")
+	_, errY := Get("yyy_t1")
+	if errX == nil || errY != nil {
+		t.Error("Error removing prefix: ", errX, errY)
+	}
+}
+
+func TestCachePush(t *testing.T) {
+	CachePush("ccc_t1", "1")
+	CachePush("ccc_t1", "2")
+	v, err := Get("ccc_t1")
+	if err != nil || len(v.(map[interface{}]struct{})) != 2 {
+		t.Error("Error in cache push: ", v)
+	}
+}
+
+func TestCount(t *testing.T) {
+	Cache("dst_A1", "1")
+	Cache("dst_A2", "2")
+	Cache("rpf_A3", "3")
+	Cache("dst_A4", "4")
+	Cache("dst_A5", "5")
+	if CountEntries("dst_") != 4 {
+		t.Error("Error countiong entries: ", CountEntries("dst_"))
 	}
 }

cache2go/response_cache.go

@@ -0,0 +1,78 @@
+package cache2go
+
+import (
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+var ErrNotFound = errors.New("NOT_FOUND")
+
+type CacheItem struct {
+	Value interface{}
+	Err   error
+}
+
+type ResponseCache struct {
+	ttl       time.Duration
+	cache     map[string]*CacheItem
+	semaphore map[string]chan bool
+	mu        sync.RWMutex
+}
+
+func NewResponseCache(ttl time.Duration) *ResponseCache {
+	return &ResponseCache{
+		ttl:       ttl,
+		cache:     make(map[string]*CacheItem),
+		semaphore: make(map[string]chan bool),
+		mu:        sync.RWMutex{},
+	}
+}
+
+func (rc *ResponseCache) Cache(key string, item *CacheItem) {
+	if rc.ttl == 0 {
+		return
+	}
+	rc.mu.Lock()
+	rc.cache[key] = item
+	if _, found := rc.semaphore[key]; found {
+		close(rc.semaphore[key])  // send release signal
+		delete(rc.semaphore, key) // delete key
+	}
+	rc.mu.Unlock()
+	go func() {
+		time.Sleep(rc.ttl)
+		rc.mu.Lock()
+		delete(rc.cache, key)
+		rc.mu.Unlock()
+	}()
+}
+
+func (rc *ResponseCache) Get(key string) (*CacheItem, error) {
+	if rc.ttl == 0 {
+		return nil, utils.ErrNotImplemented
+	}
+	rc.wait(key) // wait for other goroutine processsing this key
+	rc.mu.RLock()
+	defer rc.mu.RUnlock()
+	item, ok := rc.cache[key]
+	if !ok {
+		return nil, ErrNotFound
+	}
+	return item, nil
+}
+
+func (rc *ResponseCache) wait(key string) {
+	rc.mu.RLock()
+	lockChan, found := rc.semaphore[key]
+	rc.mu.RUnlock()
+	if found {
+		<-lockChan
+	} else {
+		rc.mu.Lock()
+		rc.semaphore[key] = make(chan bool)
+		rc.mu.Unlock()
+	}
+}

cache2go/response_cache_test.go

@@ -0,0 +1,25 @@
+package cache2go
+
+import (
+	"testing"
+	"time"
+)
+
+func TestRCacheSetGet(t *testing.T) {
+	rc := NewResponseCache(5 * time.Second)
+	rc.Cache("test", &CacheItem{Value: "best"})
+	v, err := rc.Get("test")
+	if err != nil || v.Value.(string) != "best" {
+		t.Error("Error retriving response cache: ", v, err)
+	}
+}
+
+func TestRCacheExpire(t *testing.T) {
+	rc := NewResponseCache(1 * time.Microsecond)
+	rc.Cache("test", &CacheItem{Value: "best"})
+	time.Sleep(1 * time.Millisecond)
+	_, err := rc.Get("test")
+	if err == nil {
+		t.Error("Error expiring response cache: ", err)
+	}
+}

cache2go/store.go

@@ -0,0 +1,223 @@
+//Simple caching library with expiration capabilities
+package cache2go
+
+import (
+	"strings"
+	"time"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+type cacheStore interface {
+	Put(string, interface{})
+	Append(string, interface{})
+	Get(string) (interface{}, error)
+	GetAge(string) (time.Duration, error)
+	Delete(string)
+	DeletePrefix(string)
+	CountEntriesForPrefix(string) int
+	GetAllForPrefix(string) (map[string]timestampedValue, error)
+	GetKeysForPrefix(string) []string
+}
+
+// easy to be counted exported by prefix
+type cacheDoubleStore map[string]map[string]timestampedValue
+
+func newDoubleStore() cacheDoubleStore {
+	return make(cacheDoubleStore)
+}
+
+func (cs cacheDoubleStore) Put(key string, value interface{}) {
+	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
+	if _, ok := cs[prefix]; !ok {
+		cs[prefix] = make(map[string]timestampedValue)
+	}
+	cs[prefix][key] = timestampedValue{time.Now(), value}
+}
+
+func (cs cacheDoubleStore) Append(key string, value interface{}) {
+	var elements map[interface{}]struct{} // using map for faster check if element is present
+	if v, err := cs.Get(key); err == nil {
+		elements = v.(map[interface{}]struct{})
+	} else {
+		elements = make(map[interface{}]struct{})
+	}
+	elements[value] = struct{}{}
+	cache.Put(key, elements)
+}
+
+func (cs cacheDoubleStore) Get(key string) (interface{}, error) {
+	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
+	if keyMap, ok := cs[prefix]; ok {
+		if ti, exists := keyMap[key]; exists {
+			return ti.value, nil
+		}
+	}
+	return nil, utils.ErrNotFound
+}
+
+func (cs cacheDoubleStore) GetAge(key string) (time.Duration, error) {
+	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
+	if keyMap, ok := cs[prefix]; ok {
+		if ti, exists := keyMap[key]; exists {
+			return time.Since(ti.timestamp), nil
+		}
+	}
+	return -1, utils.ErrNotFound
+}
+
+func (cs cacheDoubleStore) Delete(key string) {
+	prefix, key := key[:PREFIX_LEN], key[PREFIX_LEN:]
+	if keyMap, ok := cs[prefix]; ok {
+		delete(keyMap, key)
+	}
+}
+
+func (cs cacheDoubleStore) DeletePrefix(prefix string) {
+	delete(cs, prefix)
+}
+
+func (cs cacheDoubleStore) CountEntriesForPrefix(prefix string) int {
+	if m, ok := cs[prefix]; ok {
+		return len(m)
+	}
+	return 0
+}
+
+func (cs cacheDoubleStore) GetAllForPrefix(prefix string) (map[string]timestampedValue, error) {
+	if keyMap, ok := cs[prefix]; ok {
+		return keyMap, nil
+	}
+	return nil, utils.ErrNotFound
+}
+
+func (cs cacheDoubleStore) GetKeysForPrefix(prefix string) (keys []string) {
+	prefix, key := prefix[:PREFIX_LEN], prefix[PREFIX_LEN:]
+	if keyMap, ok := cs[prefix]; ok {
+		for iterKey := range keyMap {
+			if len(key) == 0 || strings.HasPrefix(iterKey, key) {
+				keys = append(keys, prefix+iterKey)
+			}
+		}
+	}
+	return
+}
+
+// faster to access
+type cacheSimpleStore struct {
+	cache    map[string]timestampedValue
+	counters map[string]int
+}
+
+func newSimpleStore() cacheSimpleStore {
+	return cacheSimpleStore{
+		cache:    make(map[string]timestampedValue),
+		counters: make(map[string]int),
+	}
+}
+
+func (cs cacheSimpleStore) Put(key string, value interface{}) {
+	if _, ok := cs.cache[key]; !ok {
+		// only count if the key is not already there
+		cs.count(key)
+	}
+	cs.cache[key] = timestampedValue{time.Now(), value}
+}
+
+func (cs cacheSimpleStore) Append(key string, value interface{}) {
+	var elements map[interface{}]struct{}
+	if v, err := cs.Get(key); err == nil {
+		elements = v.(map[interface{}]struct{})
+	} else {
+		elements = make(map[interface{}]struct{})
+	}
+	elements[value] = struct{}{}
+	cache.Put(key, elements)
+}
+
+func (cs cacheSimpleStore) Get(key string) (interface{}, error) {
+	if ti, exists := cs.cache[key]; exists {
+		return ti.value, nil
+	}
+	return nil, utils.ErrNotFound
+}
+
+func (cs cacheSimpleStore) GetAge(key string) (time.Duration, error) {
+	if ti, exists := cs.cache[key]; exists {
+		return time.Since(ti.timestamp), nil
+	}
+
+	return -1, utils.ErrNotFound
+}
+
+func (cs cacheSimpleStore) Delete(key string) {
+	if _, ok := cs.cache[key]; ok {
+		delete(cs.cache, key)
+		cs.descount(key)
+	}
+}
+
+func (cs cacheSimpleStore) DeletePrefix(prefix string) {
+	for key, _ := range cs.cache {
+		if strings.HasPrefix(key, prefix) {
+			delete(cs.cache, key)
+			cs.descount(key)
+		}
+	}
+}
+
+// increments the counter for the specified key prefix
+func (cs cacheSimpleStore) count(key string) {
+	if len(key) < PREFIX_LEN {
+		return
+	}
+	prefix := key[:PREFIX_LEN]
+	if _, ok := cs.counters[prefix]; ok {
+		// increase the value
+		cs.counters[prefix] += 1
+	} else {
+		cs.counters[prefix] = 1
+	}
+}
+
+// decrements the counter for the specified key prefix
+func (cs cacheSimpleStore) descount(key string) {
+	if len(key) < PREFIX_LEN {
+		return
+	}
+	prefix := key[:PREFIX_LEN]
+	if value, ok := cs.counters[prefix]; ok && value > 0 {
+		cs.counters[prefix] -= 1
+	}
+}
+
+func (cs cacheSimpleStore) CountEntriesForPrefix(prefix string) int {
+	if _, ok := cs.counters[prefix]; ok {
+		return cs.counters[prefix]
+	}
+	return 0
+}
+
+func (cs cacheSimpleStore) GetAllForPrefix(prefix string) (map[string]timestampedValue, error) {
+	result := make(map[string]timestampedValue)
+	found := false
+	for key, ti := range cs.cache {
+		if strings.HasPrefix(key, prefix) {
+			result[key[PREFIX_LEN:]] = ti
+			found = true
+		}
+	}
+	if !found {
+		return nil, utils.ErrNotFound
+	}
+	return result, nil
+}
+
+func (cs cacheSimpleStore) GetKeysForPrefix(prefix string) (keys []string) {
+	for key, _ := range cs.cache {
+		if strings.HasPrefix(key, prefix) {
+			keys = append(keys, key)
+		}
+	}
+	return
+}

calls_test.sh

@@ -0,0 +1,9 @@
+#! /usr/bin/env sh
+
+./local_test.sh
+lcl=$?
+echo 'go test github.com/cgrates/cgrates/general_tests -calls'
+go test github.com/cgrates/cgrates/general_tests -calls
+gnr=$?
+
+exit $gen && $gnr

cdrc/cdrc.go

@@ -0,0 +1,297 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"bufio"
+	"encoding/csv"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+	"gopkg.in/fsnotify.v1"
+)
+
+const (
+	CSV             = "csv"
+	FS_CSV          = "freeswitch_csv"
+	UNPAIRED_SUFFIX = ".unpaired"
+)
+
+// Populates the
+func populateStoredCdrField(cdr *engine.StoredCdr, fieldId, fieldVal, timezone string) error {
+	var err error
+	switch fieldId {
+	case utils.TOR:
+		cdr.TOR += fieldVal
+	case utils.ACCID:
+		cdr.AccId += fieldVal
+	case utils.REQTYPE:
+		cdr.ReqType += fieldVal
+	case utils.DIRECTION:
+		cdr.Direction += fieldVal
+	case utils.TENANT:
+		cdr.Tenant += fieldVal
+	case utils.CATEGORY:
+		cdr.Category += fieldVal
+	case utils.ACCOUNT:
+		cdr.Account += fieldVal
+	case utils.SUBJECT:
+		cdr.Subject += fieldVal
+	case utils.DESTINATION:
+		cdr.Destination += fieldVal
+	case utils.SETUP_TIME:
+		if cdr.SetupTime, err = utils.ParseTimeDetectLayout(fieldVal, timezone); err != nil {
+			return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
+		}
+	case utils.PDD:
+		if cdr.Pdd, err = utils.ParseDurationWithSecs(fieldVal); err != nil {
+			return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
+		}
+	case utils.ANSWER_TIME:
+		if cdr.AnswerTime, err = utils.ParseTimeDetectLayout(fieldVal, timezone); err != nil {
+			return fmt.Errorf("Cannot parse answer time field with value: %s, err: %s", fieldVal, err.Error())
+		}
+	case utils.USAGE:
+		if cdr.Usage, err = utils.ParseDurationWithSecs(fieldVal); err != nil {
+			return fmt.Errorf("Cannot parse duration field with value: %s, err: %s", fieldVal, err.Error())
+		}
+	case utils.SUPPLIER:
+		cdr.Supplier += fieldVal
+	case utils.DISCONNECT_CAUSE:
+		cdr.DisconnectCause += fieldVal
+	default: // Extra fields will not match predefined so they all show up here
+		cdr.ExtraFields[fieldId] += fieldVal
+	}
+	return nil
+}
+
+// Understands and processes a specific format of cdr (eg: .csv or .fwv)
+type RecordsProcessor interface {
+	ProcessNextRecord() ([]*engine.StoredCdr, error) // Process a single record in the CDR file, return a slice of CDRs since based on configuration we can have more templates
+}
+
+/*
+One instance  of CDRC will act on one folder.
+Common parameters within configs processed:
+ * cdrS, cdrFormat, cdrInDir, cdrOutDir, runDelay
+Parameters specific per config instance:
+ * duMultiplyFactor, cdrSourceId, cdrFilter, cdrFields
+*/
+func NewCdrc(cdrcCfgs map[string]*config.CdrcConfig, httpSkipTlsCheck bool, cdrs engine.Connector, closeChan chan struct{}, dfltTimezone string) (*Cdrc, error) {
+	var cdrcCfg *config.CdrcConfig
+	for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one
+		break
+	}
+	cdrc := &Cdrc{cdrFormat: cdrcCfg.CdrFormat, cdrInDir: cdrcCfg.CdrInDir, cdrOutDir: cdrcCfg.CdrOutDir,
+		runDelay: cdrcCfg.RunDelay, csvSep: cdrcCfg.FieldSeparator,
+		httpSkipTlsCheck: httpSkipTlsCheck, cdrcCfgs: cdrcCfgs, dfltCdrcCfg: cdrcCfg, timezone: utils.FirstNonEmpty(cdrcCfg.Timezone, dfltTimezone), cdrs: cdrs,
+		closeChan: closeChan, maxOpenFiles: make(chan struct{}, cdrcCfg.MaxOpenFiles),
+	}
+	var processFile struct{}
+	for i := 0; i < cdrcCfg.MaxOpenFiles; i++ {
+		cdrc.maxOpenFiles <- processFile // Empty initiate so we do not need to wait later when we pop
+	}
+	cdrc.cdrSourceIds = make([]string, len(cdrcCfgs))
+	cdrc.duMultiplyFactors = make([]float64, len(cdrcCfgs))
+	cdrc.cdrFilters = make([]utils.RSRFields, len(cdrcCfgs))
+	cdrc.cdrFields = make([][]*config.CfgCdrField, len(cdrcCfgs))
+	idx := 0
+	var err error
+	for _, cfg := range cdrcCfgs {
+		if idx == 0 { // Steal the config from just one instance since it should be the same for all
+			cdrc.failedCallsPrefix = cfg.FailedCallsPrefix
+			if cdrc.partialRecordsCache, err = NewPartialRecordsCache(cdrcCfg.PartialRecordCache, cdrcCfg.CdrOutDir, cdrcCfg.FieldSeparator); err != nil {
+				return nil, err
+			}
+		}
+		cdrc.cdrSourceIds[idx] = cfg.CdrSourceId
+		cdrc.duMultiplyFactors[idx] = cfg.DataUsageMultiplyFactor
+		cdrc.cdrFilters[idx] = cfg.CdrFilter
+		cdrc.cdrFields[idx] = cfg.ContentFields
+		idx += 1
+	}
+	// Before processing, make sure in and out folders exist
+	for _, dir := range []string{cdrc.cdrInDir, cdrc.cdrOutDir} {
+		if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
+			return nil, fmt.Errorf("Nonexistent folder: %s", dir)
+		}
+	}
+	cdrc.httpClient = new(http.Client)
+	return cdrc, nil
+}
+
+type Cdrc struct {
+	cdrFormat,
+	cdrInDir,
+	cdrOutDir string
+	failedCallsPrefix   string   // Configured failedCallsPrefix, used in case of flatstore CDRs
+	cdrSourceIds        []string // Should be in sync with cdrFields on indexes
+	runDelay            time.Duration
+	csvSep              rune
+	duMultiplyFactors   []float64
+	cdrFilters          []utils.RSRFields       // Should be in sync with cdrFields on indexes
+	cdrFields           [][]*config.CfgCdrField // Profiles directly connected with cdrFilters
+	httpSkipTlsCheck    bool
+	cdrcCfgs            map[string]*config.CdrcConfig // All cdrc config profiles attached to this CDRC (key will be profile instance name)
+	dfltCdrcCfg         *config.CdrcConfig
+	timezone            string
+	cdrs                engine.Connector
+	httpClient          *http.Client
+	closeChan           chan struct{}        // Used to signal config reloads when we need to span different CDRC-Client
+	maxOpenFiles        chan struct{}        // Maximum number of simultaneous files processed
+	partialRecordsCache *PartialRecordsCache // Shared between all files in the folder we process
+}
+
+// When called fires up folder monitoring, either automated via inotify or manual by sleeping between processing
+func (self *Cdrc) Run() error {
+	if self.runDelay == time.Duration(0) { // Automated via inotify
+		return self.trackCDRFiles()
+	}
+	// Not automated, process and sleep approach
+	for {
+		select {
+		case <-self.closeChan: // Exit, reinject closeChan for other CDRCs
+			engine.Logger.Info(fmt.Sprintf("<Cdrc> Shutting down CDRC on path %s.", self.cdrInDir))
+			return nil
+		default:
+		}
+		self.processCdrDir()
+		time.Sleep(self.runDelay)
+	}
+}
+
+// Watch the specified folder for file moves and parse the files on events
+func (self *Cdrc) trackCDRFiles() (err error) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return
+	}
+	defer watcher.Close()
+	err = watcher.Add(self.cdrInDir)
+	if err != nil {
+		return
+	}
+	engine.Logger.Info(fmt.Sprintf("<Cdrc> Monitoring %s for file moves.", self.cdrInDir))
+	for {
+		select {
+		case <-self.closeChan: // Exit, reinject closeChan for other CDRCs
+			engine.Logger.Info(fmt.Sprintf("<Cdrc> Shutting down CDRC on path %s.", self.cdrInDir))
+			return nil
+		case ev := <-watcher.Events:
+			if ev.Op&fsnotify.Create == fsnotify.Create && (self.cdrFormat != FS_CSV || path.Ext(ev.Name) != ".csv") {
+				go func() { //Enable async processing here
+					if err = self.processFile(ev.Name); err != nil {
+						engine.Logger.Err(fmt.Sprintf("Processing file %s, error: %s", ev.Name, err.Error()))
+					}
+				}()
+			}
+		case err := <-watcher.Errors:
+			engine.Logger.Err(fmt.Sprintf("Inotify error: %s", err.Error()))
+		}
+	}
+}
+
+// One run over the CDR folder
+func (self *Cdrc) processCdrDir() error {
+	engine.Logger.Info(fmt.Sprintf("<Cdrc> Parsing folder %s for CDR files.", self.cdrInDir))
+	filesInDir, _ := ioutil.ReadDir(self.cdrInDir)
+	for _, file := range filesInDir {
+		if self.cdrFormat != FS_CSV || path.Ext(file.Name()) != ".csv" {
+			go func() { //Enable async processing here
+				if err := self.processFile(path.Join(self.cdrInDir, file.Name())); err != nil {
+					engine.Logger.Err(fmt.Sprintf("Processing file %s, error: %s", file, err.Error()))
+				}
+			}()
+		}
+	}
+	return nil
+}
+
+// Processe file at filePath and posts the valid cdr rows out of it
+func (self *Cdrc) processFile(filePath string) error {
+	if cap(self.maxOpenFiles) != 0 { // 0 goes for no limit
+		processFile := <-self.maxOpenFiles // Queue here for maxOpenFiles
+		defer func() { self.maxOpenFiles <- processFile }()
+	}
+	_, fn := path.Split(filePath)
+	engine.Logger.Info(fmt.Sprintf("<Cdrc> Parsing: %s", filePath))
+	file, err := os.Open(filePath)
+	defer file.Close()
+	if err != nil {
+		engine.Logger.Crit(err.Error())
+		return err
+	}
+	var recordsProcessor RecordsProcessor
+	switch self.cdrFormat {
+	case CSV, FS_CSV, utils.KAM_FLATSTORE, utils.OSIPS_FLATSTORE:
+		csvReader := csv.NewReader(bufio.NewReader(file))
+		csvReader.Comma = self.csvSep
+		recordsProcessor = NewCsvRecordsProcessor(csvReader, self.cdrFormat, self.timezone, fn, self.failedCallsPrefix,
+			self.cdrSourceIds, self.duMultiplyFactors, self.cdrFilters, self.cdrFields, self.httpSkipTlsCheck, self.partialRecordsCache)
+	case utils.FWV:
+		recordsProcessor = NewFwvRecordsProcessor(file, self.cdrcCfgs, self.dfltCdrcCfg, self.httpClient, self.httpSkipTlsCheck, self.timezone)
+	default:
+		return fmt.Errorf("Unsupported CDR format: %s", self.cdrFormat)
+	}
+	procRowNr := 0
+	cdrsPosted := 0
+	timeStart := time.Now()
+	for {
+		cdrs, err := recordsProcessor.ProcessNextRecord()
+		if err != nil && err == io.EOF {
+			break
+		}
+		procRowNr += 1
+		if err != nil {
+			engine.Logger.Err(fmt.Sprintf("<Cdrc> Row %d, error: %s", procRowNr, err.Error()))
+			continue
+		}
+		for _, storedCdr := range cdrs { // Send CDRs to CDRS
+			var reply string
+			if self.dfltCdrcCfg.DryRun {
+				engine.Logger.Info(fmt.Sprintf("<Cdrc> DryRun CDR: %+v", storedCdr))
+				continue
+			}
+			if err := self.cdrs.ProcessCdr(storedCdr, &reply); err != nil {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed sending CDR, %+v, error: %s", storedCdr, err.Error()))
+			} else if reply != "OK" {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Received unexpected reply for CDR, %+v, reply: %s", storedCdr, reply))
+			} else {
+				cdrsPosted += 1
+			}
+		}
+	}
+	// Finished with file, move it to processed folder
+	newPath := path.Join(self.cdrOutDir, fn)
+	if err := os.Rename(filePath, newPath); err != nil {
+		engine.Logger.Err(err.Error())
+		return err
+	}
+	engine.Logger.Info(fmt.Sprintf("Finished processing %s, moved to %s. Total records processed: %d, CDRs posted: %d, run duration: %s",
+		fn, newPath, procRowNr, cdrsPosted, time.Now().Sub(timeStart)))
+	return nil
+}

cdrc/cdrc_local_test.go

@@ -0,0 +1,223 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+/*
+README:
+
+ Enable local tests by passing '-local' to the go test command
+ It is expected that the data folder of CGRateS exists at path /usr/share/cgrates/data or passed via command arguments.
+ Prior running the tests, create database and users by running:
+  mysql -pyourrootpwd < /usr/share/cgrates/data/storage/mysql/create_db_with_users.sql
+ What these tests do:
+  * Flush tables in storDb.
+  * Start engine with default configuration and give it some time to listen (here caching can slow down).
+  *
+*/
+
+var cfgPath string
+var cfg *config.CGRConfig
+var cdrcCfgs map[string]*config.CdrcConfig
+var cdrcCfg *config.CdrcConfig
+
+var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, not by default.") // This flag will be passed here via "go test -local" args
+var dataDir = flag.String("data_dir", "/usr/share/cgrates", "CGR data dir path here")
+var storDbType = flag.String("stordb_type", "mysql", "The type of the storDb database <mysql>")
+var waitRater = flag.Int("wait_rater", 300, "Number of miliseconds to wait for rater to start and cache")
+
+var fileContent1 = `accid11,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1
+accid12,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1
+dummy_data
+accid13,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1
+`
+
+var fileContent2 = `accid21,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1
+accid22,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1
+#accid1,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1
+accid23,prepaid,out,cgrates.org,call,1001,1001,+4986517174963,2013-02-03 19:54:00,62,supplier1,172.16.1.1`
+
+var fileContent3 = `accid31;prepaid;out;cgrates.org;call;1001;1001;+4986517174963;2013-02-03 19:54:00;62;supplier1;172.16.1.1
+accid32;prepaid;out;cgrates.org;call;1001;1001;+4986517174963;2013-02-03 19:54:00;62;supplier1;172.16.1.1
+#accid1;prepaid;out;cgrates.org;call;1001;1001;+4986517174963;2013-02-03 19:54:00;62;supplier1;172.16.1.1
+accid33;prepaid;out;cgrates.org;call;1001;1001;+4986517174963;2013-02-03 19:54:00;62;supplier1;172.16.1.1`
+
+func startEngine() error {
+	enginePath, err := exec.LookPath("cgr-engine")
+	if err != nil {
+		return errors.New("Cannot find cgr-engine executable")
+	}
+	stopEngine()
+	engine := exec.Command(enginePath, "-config", cfgPath)
+	if err := engine.Start(); err != nil {
+		return fmt.Errorf("Cannot start cgr-engine: %s", err.Error())
+	}
+	time.Sleep(time.Duration(*waitRater) * time.Millisecond) // Give time to rater to fire up
+	return nil
+}
+
+func stopEngine() error {
+	exec.Command("pkill", "cgr-engine").Run() // Just to make sure another one is not running, bit brutal maybe we can fine tune it
+	return nil
+}
+
+// Need it here and not in init since Travis has no possibility to load local file
+func TestLoadConfigt(*testing.T) {
+	if !*testLocal {
+		return
+	}
+	cfgPath = path.Join(*dataDir, "conf", "samples", "apier")
+	cfg, _ = config.NewCGRConfigFromFolder(cfgPath)
+	if len(cfg.CdrcProfiles) > 0 {
+		cdrcCfgs = cfg.CdrcProfiles["/var/log/cgrates/cdrc/in"]
+	}
+}
+
+func TestEmptyTables(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if *storDbType != utils.MYSQL {
+		t.Fatal("Unsupported storDbType")
+	}
+	var mysql *engine.MySQLStorage
+	if d, err := engine.NewMySQLStorage(cfg.StorDBHost, cfg.StorDBPort, cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns); err != nil {
+		t.Fatal("Error on opening database connection: ", err)
+	} else {
+		mysql = d.(*engine.MySQLStorage)
+	}
+	for _, scriptName := range []string{utils.CREATE_CDRS_TABLES_SQL, utils.CREATE_TARIFFPLAN_TABLES_SQL} {
+		if err := mysql.CreateTablesFromScript(path.Join(*dataDir, "storage", *storDbType, scriptName)); err != nil {
+			t.Fatal("Error on mysql creation: ", err.Error())
+			return // No point in going further
+		}
+	}
+	for _, tbl := range []string{utils.TBL_CDRS_PRIMARY, utils.TBL_CDRS_EXTRA} {
+		if _, err := mysql.Db.Query(fmt.Sprintf("SELECT 1 from %s", tbl)); err != nil {
+			t.Fatal(err.Error())
+		}
+	}
+}
+
+// Creates cdr files and starts the engine
+func TestCreateCdrFiles(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if cdrcCfgs == nil {
+		t.Fatal("Empty default cdrc configuration")
+	}
+	for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one
+		break
+	}
+	if err := os.RemoveAll(cdrcCfg.CdrInDir); err != nil {
+		t.Fatal("Error removing folder: ", cdrcCfg.CdrInDir, err)
+	}
+	if err := os.MkdirAll(cdrcCfg.CdrInDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", cdrcCfg.CdrInDir, err)
+	}
+	if err := os.RemoveAll(cdrcCfg.CdrOutDir); err != nil {
+		t.Fatal("Error removing folder: ", cdrcCfg.CdrOutDir, err)
+	}
+	if err := os.MkdirAll(cdrcCfg.CdrOutDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", cdrcCfg.CdrOutDir, err)
+	}
+	if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file1.csv"), []byte(fileContent1), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file2.csv"), []byte(fileContent2), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+
+}
+
+func TestProcessCdrDir(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var cdrcCfg *config.CdrcConfig
+	for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one
+		break
+	}
+	if cdrcCfg.Cdrs == utils.INTERNAL { // For now we only test over network
+		cdrcCfg.Cdrs = "127.0.0.1:2013"
+	}
+	if err := startEngine(); err != nil {
+		t.Fatal(err.Error())
+	}
+	cdrc, err := NewCdrc(cdrcCfgs, true, nil, make(chan struct{}), "")
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	if err := cdrc.processCdrDir(); err != nil {
+		t.Error(err)
+	}
+	stopEngine()
+}
+
+// Creates cdr files and starts the engine
+func TestCreateCdr3File(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := os.RemoveAll(cdrcCfg.CdrInDir); err != nil {
+		t.Fatal("Error removing folder: ", cdrcCfg.CdrInDir, err)
+	}
+	if err := os.MkdirAll(cdrcCfg.CdrInDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", cdrcCfg.CdrInDir, err)
+	}
+	if err := ioutil.WriteFile(path.Join(cdrcCfg.CdrInDir, "file3.csv"), []byte(fileContent3), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+}
+
+func TestProcessCdr3Dir(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if cdrcCfg.Cdrs == utils.INTERNAL { // For now we only test over network
+		cdrcCfg.Cdrs = "127.0.0.1:2013"
+	}
+	if err := startEngine(); err != nil {
+		t.Fatal(err.Error())
+	}
+	cdrc, err := NewCdrc(cdrcCfgs, true, nil, make(chan struct{}), "")
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	if err := cdrc.processCdrDir(); err != nil {
+		t.Error(err)
+	}
+	stopEngine()
+}

cdrc/cdrc_test.go

@@ -0,0 +1,318 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+/*
+func TestNewPartialFlatstoreRecord(t *testing.T) {
+	ePr := &PartialFlatstoreRecord{Method: "INVITE", AccId: "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:02daec40c548625ac", Timestamp: time.Date(2015, 7, 9, 15, 6, 48, 0, time.UTC),
+		Values: []string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454408", "*prepaid", "1001", "1002", "", "3401:2069362475"}}
+	if pr, err := NewPartialFlatstoreRecord(ePr.Values); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(ePr, pr) {
+		t.Errorf("Expecting: %+v, received: %+v", ePr, pr)
+	}
+	if _, err := NewPartialFlatstoreRecord([]string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK"}); err == nil || err.Error() != "MISSING_IE" {
+		t.Error(err)
+	}
+}
+*/
+
+/*
+func TestOsipsFlatstoreCdrs(t *testing.T) {
+	flatstoreCdrs := `
+INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475
+BYE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454410|||||3401:2069362475
+INVITE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454647|*postpaid|1002|1001||1877:893549741
+BYE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454651|||||1877:893549741
+INVITE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454657|*prepaid|1001|1002||2407:1884881533
+BYE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454661|||||2407:1884881533
+INVITE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454690|*prepaid|1001|1002||3099:1909036290
+BYE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454692|||||3099:1909036290
+`
+
+	eCdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{
+			CgrId:           "e61034c34148a7c4f40623e00ca5e551d1408bf3",
+			TOR:             utils.VOICE,
+			AccId:           "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:02daec40c548625ac",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 06, 48, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 06, 48, 0, time.UTC),
+			Usage:           time.Duration(2) * time.Second,
+			DisconnectCause: "200 OK",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "3401:2069362475",
+			},
+			Cost: -1,
+		},
+		&engine.StoredCdr{
+			CgrId:           "3ed64a28190e20ac8a6fd8fd48cb23efbfeb7a17",
+			TOR:             utils.VOICE,
+			AccId:           "214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0f9d3d5c3c863a6e3",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_POSTPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1002",
+			Subject:         "1002",
+			Destination:     "1001",
+			SetupTime:       time.Date(2015, 7, 9, 15, 10, 47, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 10, 47, 0, time.UTC),
+			Usage:           time.Duration(4) * time.Second,
+			DisconnectCause: "200 OK",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "1877:893549741",
+			},
+			Cost: -1,
+		},
+		&engine.StoredCdr{
+			CgrId:           "f2f8d9341adfbbe1836b22f75182142061ef3d20",
+			TOR:             utils.VOICE,
+			AccId:           "3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:036e39a542d996f9",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 10, 57, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 10, 57, 0, time.UTC),
+			Usage:           time.Duration(4) * time.Second,
+			DisconnectCause: "200 OK",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "2407:1884881533",
+			},
+			Cost: -1,
+		},
+		&engine.StoredCdr{
+			CgrId:           "ccf05e7e3b9db9d2370bcbe316817447dba7df54",
+			TOR:             utils.VOICE,
+			AccId:           "a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:03111f3c949ca4c42",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 11, 30, 0, time.UTC), //2015-07-09T17:11:30+02:00
+			AnswerTime:      time.Date(2015, 7, 9, 15, 11, 30, 0, time.UTC),
+			Usage:           time.Duration(2) * time.Second,
+			DisconnectCause: "200 OK",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "3099:1909036290",
+			},
+			Cost: -1,
+		},
+	}
+
+	cdrFields := [][]*config.CfgCdrField{[]*config.CfgCdrField{
+		&config.CfgCdrField{Tag: "Tor", Type: utils.CDRFIELD, CdrFieldId: utils.TOR, Value: utils.ParseRSRFieldsMustCompile("^*voice", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "AccId", Type: utils.CDRFIELD, CdrFieldId: utils.ACCID, Mandatory: true},
+		&config.CfgCdrField{Tag: "ReqType", Type: utils.CDRFIELD, CdrFieldId: utils.REQTYPE, Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Tenant", Type: utils.CDRFIELD, CdrFieldId: utils.TENANT, Value: utils.ParseRSRFieldsMustCompile("^cgrates.org", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Category", Type: utils.CDRFIELD, CdrFieldId: utils.CATEGORY, Value: utils.ParseRSRFieldsMustCompile("^call", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Account", Type: utils.CDRFIELD, CdrFieldId: utils.ACCOUNT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Subject", Type: utils.CDRFIELD, CdrFieldId: utils.SUBJECT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Destination", Type: utils.CDRFIELD, CdrFieldId: utils.DESTINATION, Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "SetupTime", Type: utils.CDRFIELD, CdrFieldId: utils.SETUP_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "AnswerTime", Type: utils.CDRFIELD, CdrFieldId: utils.ANSWER_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Duration", Type: utils.CDRFIELD, CdrFieldId: utils.USAGE, Mandatory: true},
+		&config.CfgCdrField{Tag: "DisconnectCause", Type: utils.CDRFIELD, CdrFieldId: utils.DISCONNECT_CAUSE, Value: utils.ParseRSRFieldsMustCompile("4;^ ;5", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "DialogId", Type: utils.CDRFIELD, CdrFieldId: "DialogIdentifier", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP)},
+	}}
+	cdrc := &Cdrc{CdrFormat: utils.OSIPS_FLATSTORE, cdrSourceIds: []string{"TEST_CDRC"}, failedCallsPrefix: "missed_calls",
+		cdrFields: cdrFields, partialRecords: make(map[string]map[string]*PartialFlatstoreRecord),
+		guard: engine.NewGuardianLock()}
+	cdrsContent := bytes.NewReader([]byte(flatstoreCdrs))
+	csvReader := csv.NewReader(cdrsContent)
+	csvReader.Comma = '|'
+	cdrs := make([]*engine.StoredCdr, 0)
+	recNrs := 0
+	for {
+		recNrs++
+		cdrCsv, err := csvReader.Read()
+		if err != nil && err == io.EOF {
+			break // End of file
+		} else if err != nil {
+			t.Error("Unexpected error:", err)
+		}
+		record, err := cdrc.processPartialRecord(cdrCsv, "dummyfilename")
+		if err != nil {
+			t.Error(err)
+		}
+		if record == nil {
+			continue // Partial record
+		}
+		if storedCdr, err := cdrc.recordToStoredCdr(record, 0); err != nil {
+			t.Error(err)
+		} else if storedCdr != nil {
+			cdrs = append(cdrs, storedCdr)
+		}
+	}
+	if !reflect.DeepEqual(eCdrs, cdrs) {
+		t.Errorf("Expecting: %+v, received: %+v", eCdrs, cdrs)
+	}
+
+}
+
+func TestOsipsFlatstoreMissedCdrs(t *testing.T) {
+	flatstoreCdrs := `
+INVITE|ef6c6256|da501581|0bfdd176d1b93e7df3de5c6f4873ee04@0:0:0:0:0:0:0:0|487|Request Terminated|1436454643|*prepaid|1001|1002||1224:339382783
+INVITE|7905e511||81880da80a94bda81b425b09009e055c@0:0:0:0:0:0:0:0|404|Not Found|1436454668|*prepaid|1001|1002||1980:1216490844
+INVITE|324cb497|d4af7023|8deaadf2ae9a17809a391f05af31afb0@0:0:0:0:0:0:0:0|486|Busy here|1436454687|*postpaid|1002|1001||474:130115066
+`
+	eCdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{
+			CgrId:           "1c20aa6543a5a30d26b2354ae79e1f5fb720e8e5",
+			TOR:             utils.VOICE,
+			AccId:           "0bfdd176d1b93e7df3de5c6f4873ee04@0:0:0:0:0:0:0:0ef6c6256da501581",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 10, 43, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 10, 43, 0, time.UTC),
+			Usage:           0,
+			DisconnectCause: "487 Request Terminated",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "1224:339382783",
+			},
+			Cost: -1,
+		},
+		&engine.StoredCdr{
+			CgrId:           "054ab7c6c7fe6dc4a72f34e270027fa2aa930a58",
+			TOR:             utils.VOICE,
+			AccId:           "81880da80a94bda81b425b09009e055c@0:0:0:0:0:0:0:07905e511",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_PREPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1001",
+			Subject:         "1001",
+			Destination:     "1002",
+			SetupTime:       time.Date(2015, 7, 9, 15, 11, 8, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 11, 8, 0, time.UTC),
+			Usage:           0,
+			DisconnectCause: "404 Not Found",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "1980:1216490844",
+			},
+			Cost: -1,
+		},
+		&engine.StoredCdr{
+			CgrId:           "d49ea63d1655b15149336004629f1cadd1434b89",
+			TOR:             utils.VOICE,
+			AccId:           "8deaadf2ae9a17809a391f05af31afb0@0:0:0:0:0:0:0:0324cb497d4af7023",
+			CdrHost:         "0.0.0.0",
+			CdrSource:       "TEST_CDRC",
+			ReqType:         utils.META_POSTPAID,
+			Direction:       "*out",
+			Tenant:          "cgrates.org",
+			Category:        "call",
+			Account:         "1002",
+			Subject:         "1002",
+			Destination:     "1001",
+			SetupTime:       time.Date(2015, 7, 9, 15, 11, 27, 0, time.UTC),
+			AnswerTime:      time.Date(2015, 7, 9, 15, 11, 27, 0, time.UTC),
+			Usage:           0,
+			DisconnectCause: "486 Busy here",
+			ExtraFields: map[string]string{
+				"DialogIdentifier": "474:130115066",
+			},
+			Cost: -1,
+		},
+	}
+
+	cdrFields := [][]*config.CfgCdrField{[]*config.CfgCdrField{
+		&config.CfgCdrField{Tag: "Tor", Type: utils.CDRFIELD, CdrFieldId: utils.TOR, Value: utils.ParseRSRFieldsMustCompile("^*voice", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "AccId", Type: utils.CDRFIELD, CdrFieldId: utils.ACCID, Mandatory: true},
+		&config.CfgCdrField{Tag: "ReqType", Type: utils.CDRFIELD, CdrFieldId: utils.REQTYPE, Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Direction", Type: utils.CDRFIELD, CdrFieldId: utils.DIRECTION, Value: utils.ParseRSRFieldsMustCompile("^*out", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Tenant", Type: utils.CDRFIELD, CdrFieldId: utils.TENANT, Value: utils.ParseRSRFieldsMustCompile("^cgrates.org", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Category", Type: utils.CDRFIELD, CdrFieldId: utils.CATEGORY, Value: utils.ParseRSRFieldsMustCompile("^call", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Account", Type: utils.CDRFIELD, CdrFieldId: utils.ACCOUNT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Subject", Type: utils.CDRFIELD, CdrFieldId: utils.SUBJECT, Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Destination", Type: utils.CDRFIELD, CdrFieldId: utils.DESTINATION, Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "SetupTime", Type: utils.CDRFIELD, CdrFieldId: utils.SETUP_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "AnswerTime", Type: utils.CDRFIELD, CdrFieldId: utils.ANSWER_TIME, Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "Usage", Type: utils.CDRFIELD, CdrFieldId: utils.USAGE, Mandatory: true},
+		&config.CfgCdrField{Tag: "DisconnectCause", Type: utils.CDRFIELD, CdrFieldId: utils.DISCONNECT_CAUSE, Value: utils.ParseRSRFieldsMustCompile("4;^ ;5", utils.INFIELD_SEP), Mandatory: true},
+		&config.CfgCdrField{Tag: "DialogId", Type: utils.CDRFIELD, CdrFieldId: "DialogIdentifier", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP)},
+	}}
+	cdrc := &Cdrc{CdrFormat: utils.OSIPS_FLATSTORE, cdrSourceIds: []string{"TEST_CDRC"}, failedCallsPrefix: "missed_calls",
+		cdrFields: cdrFields, partialRecords: make(map[string]map[string]*PartialFlatstoreRecord),
+		guard: engine.NewGuardianLock()}
+	cdrsContent := bytes.NewReader([]byte(flatstoreCdrs))
+	csvReader := csv.NewReader(cdrsContent)
+	csvReader.Comma = '|'
+	cdrs := make([]*engine.StoredCdr, 0)
+	recNrs := 0
+	for {
+		recNrs++
+		cdrCsv, err := csvReader.Read()
+		if err != nil && err == io.EOF {
+			break // End of file
+		} else if err != nil {
+			t.Error("Unexpected error:", err)
+		}
+		record, err := cdrc.processPartialRecord(cdrCsv, "missed_calls_1.log")
+		if err != nil {
+			t.Error(err)
+		}
+		if record == nil {
+			continue // Partial record
+		}
+		if storedCdr, err := cdrc.recordToStoredCdr(record, 0); err != nil {
+			t.Error(err)
+		} else if storedCdr != nil {
+			cdrs = append(cdrs, storedCdr)
+		}
+	}
+	if !reflect.DeepEqual(eCdrs, cdrs) {
+		t.Errorf("Expecting: %+v, received: %+v", eCdrs, cdrs)
+	}
+
+}
+*/

cdrc/csv.go

@@ -0,0 +1,335 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func NewPartialFlatstoreRecord(record []string, timezone string) (*PartialFlatstoreRecord, error) {
+	if len(record) < 7 {
+		return nil, errors.New("MISSING_IE")
+	}
+	pr := &PartialFlatstoreRecord{Method: record[0], AccId: record[3] + record[1] + record[2], Values: record}
+	var err error
+	if pr.Timestamp, err = utils.ParseTimeDetectLayout(record[6], timezone); err != nil {
+		return nil, err
+	}
+	return pr, nil
+}
+
+// This is a partial record received from Flatstore, can be INVITE or BYE and it needs to be paired in order to produce duration
+type PartialFlatstoreRecord struct {
+	Method    string    // INVITE or BYE
+	AccId     string    // Copute here the AccId
+	Timestamp time.Time // Timestamp of the event, as written by db_flastore module
+	Values    []string  // Can contain original values or updated via UpdateValues
+}
+
+// Pairs INVITE and BYE into final record containing as last element the duration
+func pairToRecord(part1, part2 *PartialFlatstoreRecord) ([]string, error) {
+	var invite, bye *PartialFlatstoreRecord
+	if part1.Method == "INVITE" {
+		invite = part1
+	} else if part2.Method == "INVITE" {
+		invite = part2
+	} else {
+		return nil, errors.New("MISSING_INVITE")
+	}
+	if part1.Method == "BYE" {
+		bye = part1
+	} else if part2.Method == "BYE" {
+		bye = part2
+	} else {
+		return nil, errors.New("MISSING_BYE")
+	}
+	if len(invite.Values) != len(bye.Values) {
+		return nil, errors.New("INCONSISTENT_VALUES_LENGTH")
+	}
+	record := invite.Values
+	for idx := range record {
+		switch idx {
+		case 0, 1, 2, 3, 6: // Leave these values as they are
+		case 4, 5:
+			record[idx] = bye.Values[idx] // Update record with status from bye
+		default:
+			if bye.Values[idx] != "" { // Any value higher than 6 is dynamically inserted, overwrite if non empty
+				record[idx] = bye.Values[idx]
+			}
+
+		}
+	}
+	callDur := bye.Timestamp.Sub(invite.Timestamp)
+	record = append(record, strconv.FormatFloat(callDur.Seconds(), 'f', -1, 64))
+	return record, nil
+}
+
+func NewPartialRecordsCache(ttl time.Duration, cdrOutDir string, csvSep rune) (*PartialRecordsCache, error) {
+	return &PartialRecordsCache{ttl: ttl, cdrOutDir: cdrOutDir, csvSep: csvSep,
+		partialRecords: make(map[string]map[string]*PartialFlatstoreRecord), guard: engine.NewGuardianLock()}, nil
+}
+
+type PartialRecordsCache struct {
+	ttl            time.Duration
+	cdrOutDir      string
+	csvSep         rune
+	partialRecords map[string]map[string]*PartialFlatstoreRecord // [FileName"][AccId]*PartialRecord
+	guard          *engine.GuardianLock
+}
+
+// Dumps the cache into a .unpaired file in the outdir and cleans cache after
+func (self *PartialRecordsCache) dumpUnpairedRecords(fileName string) error {
+	_, err := self.guard.Guard(func() (interface{}, error) {
+		if len(self.partialRecords[fileName]) != 0 { // Only write the file if there are records in the cache
+			unpairedFilePath := path.Join(self.cdrOutDir, fileName+UNPAIRED_SUFFIX)
+			fileOut, err := os.Create(unpairedFilePath)
+			if err != nil {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed creating %s, error: %s", unpairedFilePath, err.Error()))
+				return nil, err
+			}
+			csvWriter := csv.NewWriter(fileOut)
+			csvWriter.Comma = self.csvSep
+			for _, pr := range self.partialRecords[fileName] {
+				if err := csvWriter.Write(pr.Values); err != nil {
+					engine.Logger.Err(fmt.Sprintf("<Cdrc> Failed writing unpaired record %v to file: %s, error: %s", pr, unpairedFilePath, err.Error()))
+					return nil, err
+				}
+			}
+			csvWriter.Flush()
+		}
+		delete(self.partialRecords, fileName)
+		return nil, nil
+	}, 0, fileName)
+	return err
+}
+
+// Search in cache and return the partial record with accountind id defined, prefFilename is searched at beginning because of better match probability
+func (self *PartialRecordsCache) GetPartialRecord(accId, prefFileName string) (string, *PartialFlatstoreRecord) {
+	var cachedFilename string
+	var cachedPartial *PartialFlatstoreRecord
+	checkCachedFNames := []string{prefFileName} // Higher probability to match as firstFileName
+	for fName := range self.partialRecords {
+		if fName != prefFileName {
+			checkCachedFNames = append(checkCachedFNames, fName)
+		}
+	}
+	for _, fName := range checkCachedFNames { // Need to lock them individually
+		self.guard.Guard(func() (interface{}, error) {
+			var hasPartial bool
+			if cachedPartial, hasPartial = self.partialRecords[fName][accId]; hasPartial {
+				cachedFilename = fName
+			}
+			return nil, nil
+		}, 0, fName)
+		if cachedPartial != nil {
+			break
+		}
+	}
+	return cachedFilename, cachedPartial
+}
+
+func (self *PartialRecordsCache) CachePartial(fileName string, pr *PartialFlatstoreRecord) {
+	self.guard.Guard(func() (interface{}, error) {
+		if fileMp, hasFile := self.partialRecords[fileName]; !hasFile {
+			self.partialRecords[fileName] = map[string]*PartialFlatstoreRecord{pr.AccId: pr}
+			if self.ttl != 0 { // Schedule expiry/dump of the just created entry in cache
+				go func() {
+					time.Sleep(self.ttl)
+					self.dumpUnpairedRecords(fileName)
+				}()
+			}
+		} else if _, hasAccId := fileMp[pr.AccId]; !hasAccId {
+			self.partialRecords[fileName][pr.AccId] = pr
+		}
+		return nil, nil
+	}, 0, fileName)
+}
+
+func (self *PartialRecordsCache) UncachePartial(fileName string, pr *PartialFlatstoreRecord) {
+	self.guard.Guard(func() (interface{}, error) {
+		delete(self.partialRecords[fileName], pr.AccId) // Remove the record out of cache
+		return nil, nil
+	}, 0, fileName)
+}
+
+func NewCsvRecordsProcessor(csvReader *csv.Reader, cdrFormat, timezone, fileName, failedCallsPrefix string,
+	cdrSourceIds []string, duMultiplyFactors []float64, cdrFilters []utils.RSRFields, cdrFields [][]*config.CfgCdrField,
+	httpSkipTlsCheck bool, partialRecordsCache *PartialRecordsCache) *CsvRecordsProcessor {
+	return &CsvRecordsProcessor{csvReader: csvReader, cdrFormat: cdrFormat, timezone: timezone, fileName: fileName,
+		failedCallsPrefix: failedCallsPrefix, cdrSourceIds: cdrSourceIds,
+		duMultiplyFactors: duMultiplyFactors, cdrFilters: cdrFilters, cdrFields: cdrFields,
+		httpSkipTlsCheck: httpSkipTlsCheck, partialRecordsCache: partialRecordsCache}
+
+}
+
+type CsvRecordsProcessor struct {
+	csvReader           *csv.Reader
+	cdrFormat           string
+	timezone            string // Timezone for CDRs which are not clearly specifying it
+	fileName            string
+	failedCallsPrefix   string
+	cdrSourceIds        []string // Should be in sync with cdrFields on indexes
+	duMultiplyFactors   []float64
+	cdrFilters          []utils.RSRFields       // Should be in sync with cdrFields on indexes
+	cdrFields           [][]*config.CfgCdrField // Profiles directly connected with cdrFilters
+	httpSkipTlsCheck    bool
+	partialRecordsCache *PartialRecordsCache // Shared by cdrc so we can cache for all files in a folder
+}
+
+func (self *CsvRecordsProcessor) ProcessNextRecord() ([]*engine.StoredCdr, error) {
+	record, err := self.csvReader.Read()
+	if err != nil {
+		return nil, err
+	}
+	if utils.IsSliceMember([]string{utils.KAM_FLATSTORE, utils.OSIPS_FLATSTORE}, self.cdrFormat) {
+		if record, err = self.processPartialRecord(record); err != nil {
+			return nil, err
+		} else if record == nil {
+			return nil, nil // Due to partial, none returned
+		}
+	}
+	// Record was overwriten with complete information out of cache
+	return self.processRecord(record)
+}
+
+// Processes a single partial record for flatstore CDRs
+func (self *CsvRecordsProcessor) processPartialRecord(record []string) ([]string, error) {
+	if strings.HasPrefix(self.fileName, self.failedCallsPrefix) { // Use the first index since they should be the same in all configs
+		record = append(record, "0") // Append duration 0 for failed calls flatstore CDR and do not process it further
+		return record, nil
+	}
+	pr, err := NewPartialFlatstoreRecord(record, self.timezone)
+	if err != nil {
+		return nil, err
+	}
+	// Retrieve and complete the record from cache
+	cachedFilename, cachedPartial := self.partialRecordsCache.GetPartialRecord(pr.AccId, self.fileName)
+	if cachedPartial == nil { // Not cached, do it here and stop processing
+		self.partialRecordsCache.CachePartial(self.fileName, pr)
+		return nil, nil
+	}
+	pairedRecord, err := pairToRecord(cachedPartial, pr)
+	if err != nil {
+		return nil, err
+	}
+	self.partialRecordsCache.UncachePartial(cachedFilename, pr)
+	return pairedRecord, nil
+}
+
+// Takes the record from a slice and turns it into StoredCdrs, posting them to the cdrServer
+func (self *CsvRecordsProcessor) processRecord(record []string) ([]*engine.StoredCdr, error) {
+	recordCdrs := make([]*engine.StoredCdr, 0) // More CDRs based on the number of filters and field templates
+	for idx := range self.cdrFields {          // cdrFields coming from more templates will produce individual storCdr records
+		// Make sure filters are matching
+		filterBreak := false
+		for _, rsrFilter := range self.cdrFilters[idx] {
+			if rsrFilter == nil { // Nil filter does not need to match anything
+				continue
+			}
+			if cfgFieldIdx, _ := strconv.Atoi(rsrFilter.Id); len(record) <= cfgFieldIdx {
+				return nil, fmt.Errorf("Ignoring record: %v - cannot compile filter %+v", record, rsrFilter)
+			} else if !rsrFilter.FilterPasses(record[cfgFieldIdx]) {
+				filterBreak = true
+				break
+			}
+		}
+		if filterBreak { // Stop importing cdrc fields profile due to non matching filter
+			continue
+		}
+		if storedCdr, err := self.recordToStoredCdr(record, idx); err != nil {
+			return nil, fmt.Errorf("Failed converting to StoredCdr, error: %s", err.Error())
+		} else {
+			recordCdrs = append(recordCdrs, storedCdr)
+		}
+	}
+	return recordCdrs, nil
+}
+
+// Takes the record out of csv and turns it into storedCdr which can be processed by CDRS
+func (self *CsvRecordsProcessor) recordToStoredCdr(record []string, cfgIdx int) (*engine.StoredCdr, error) {
+	storedCdr := &engine.StoredCdr{CdrHost: "0.0.0.0", CdrSource: self.cdrSourceIds[cfgIdx], ExtraFields: make(map[string]string), Cost: -1}
+	var err error
+	var lazyHttpFields []*config.CfgCdrField
+	for _, cdrFldCfg := range self.cdrFields[cfgIdx] {
+		if utils.IsSliceMember([]string{utils.KAM_FLATSTORE, utils.OSIPS_FLATSTORE}, self.cdrFormat) { // Hardcode some values in case of flatstore
+			switch cdrFldCfg.CdrFieldId {
+			case utils.ACCID:
+				cdrFldCfg.Value = utils.ParseRSRFieldsMustCompile("3;1;2", utils.INFIELD_SEP) // in case of flatstore, accounting id is made up out of callid, from_tag and to_tag
+			case utils.USAGE:
+				cdrFldCfg.Value = utils.ParseRSRFieldsMustCompile(strconv.Itoa(len(record)-1), utils.INFIELD_SEP) // in case of flatstore, last element will be the duration computed by us
+			}
+
+		}
+		var fieldVal string
+		if cdrFldCfg.Type == utils.CDRFIELD {
+			for _, cfgFieldRSR := range cdrFldCfg.Value {
+				if cfgFieldRSR.IsStatic() {
+					fieldVal += cfgFieldRSR.ParseValue("")
+				} else { // Dynamic value extracted using index
+					if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx {
+						return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cdrFldCfg.Tag)
+					} else {
+						fieldVal += cfgFieldRSR.ParseValue(record[cfgFieldIdx])
+					}
+				}
+			}
+		} else if cdrFldCfg.Type == utils.HTTP_POST {
+			lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of storedCdr to http server
+		} else {
+			return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type)
+		}
+		if err := populateStoredCdrField(storedCdr, cdrFldCfg.CdrFieldId, fieldVal, self.timezone); err != nil {
+			return nil, err
+		}
+	}
+	storedCdr.CgrId = utils.Sha1(storedCdr.AccId, storedCdr.SetupTime.UTC().String())
+	if storedCdr.TOR == utils.DATA && self.duMultiplyFactors[cfgIdx] != 0 {
+		storedCdr.Usage = time.Duration(float64(storedCdr.Usage.Nanoseconds()) * self.duMultiplyFactors[cfgIdx])
+	}
+	for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields
+		var outValByte []byte
+		var fieldVal, httpAddr string
+		for _, rsrFld := range httpFieldCfg.Value {
+			httpAddr += rsrFld.ParseValue("")
+		}
+		if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, storedCdr); err != nil && httpFieldCfg.Mandatory {
+			return nil, err
+		} else {
+			fieldVal = string(outValByte)
+			if len(fieldVal) == 0 && httpFieldCfg.Mandatory {
+				return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag)
+			}
+			if err := populateStoredCdrField(storedCdr, httpFieldCfg.CdrFieldId, fieldVal, self.timezone); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return storedCdr, nil
+}

cdrc/csv_test.go

@@ -0,0 +1,151 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func TestCsvRecordForkCdr(t *testing.T) {
+	cgrConfig, _ := config.NewDefaultCGRConfig()
+	cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][utils.META_DEFAULT]
+	cdrcConfig.ContentFields = append(cdrcConfig.ContentFields, &config.CfgCdrField{Tag: "SupplierTest", Type: utils.CDRFIELD, CdrFieldId: "supplier", Value: []*utils.RSRField{&utils.RSRField{Id: "14"}}})
+	cdrcConfig.ContentFields = append(cdrcConfig.ContentFields, &config.CfgCdrField{Tag: "DisconnectCauseTest", Type: utils.CDRFIELD, CdrFieldId: utils.DISCONNECT_CAUSE,
+		Value: []*utils.RSRField{&utils.RSRField{Id: "16"}}})
+	csvProcessor := &CsvRecordsProcessor{cdrFormat: CSV, cdrSourceIds: []string{"TEST_CDRC"}, cdrFields: [][]*config.CfgCdrField{cdrcConfig.ContentFields}}
+	cdrRow := []string{"firstField", "secondField"}
+	_, err := csvProcessor.recordToStoredCdr(cdrRow, 0)
+	if err == nil {
+		t.Error("Failed to corectly detect missing fields from record")
+	}
+	cdrRow = []string{"ignored", "ignored", utils.VOICE, "acc1", utils.META_PREPAID, "*out", "cgrates.org", "call", "1001", "1001", "+4986517174963",
+		"2013-02-03 19:50:00", "2013-02-03 19:54:00", "62", "supplier1", "172.16.1.1", "NORMAL_DISCONNECT"}
+	rtCdr, err := csvProcessor.recordToStoredCdr(cdrRow, 0)
+	if err != nil {
+		t.Error("Failed to parse CDR in rated cdr", err)
+	}
+	expectedCdr := &engine.StoredCdr{
+		CgrId:           utils.Sha1(cdrRow[3], time.Date(2013, 2, 3, 19, 50, 0, 0, time.UTC).String()),
+		TOR:             cdrRow[2],
+		AccId:           cdrRow[3],
+		CdrHost:         "0.0.0.0", // Got it over internal interface
+		CdrSource:       "TEST_CDRC",
+		ReqType:         cdrRow[4],
+		Direction:       cdrRow[5],
+		Tenant:          cdrRow[6],
+		Category:        cdrRow[7],
+		Account:         cdrRow[8],
+		Subject:         cdrRow[9],
+		Destination:     cdrRow[10],
+		SetupTime:       time.Date(2013, 2, 3, 19, 50, 0, 0, time.UTC),
+		AnswerTime:      time.Date(2013, 2, 3, 19, 54, 0, 0, time.UTC),
+		Usage:           time.Duration(62) * time.Second,
+		Supplier:        "supplier1",
+		DisconnectCause: "NORMAL_DISCONNECT",
+		ExtraFields:     map[string]string{},
+		Cost:            -1,
+	}
+	if !reflect.DeepEqual(expectedCdr, rtCdr) {
+		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
+	}
+}
+
+func TestCsvDataMultiplyFactor(t *testing.T) {
+	cdrFields := []*config.CfgCdrField{&config.CfgCdrField{Tag: "TORField", Type: utils.CDRFIELD, CdrFieldId: "tor", Value: []*utils.RSRField{&utils.RSRField{Id: "0"}}},
+		&config.CfgCdrField{Tag: "UsageField", Type: utils.CDRFIELD, CdrFieldId: "usage", Value: []*utils.RSRField{&utils.RSRField{Id: "1"}}}}
+	csvProcessor := &CsvRecordsProcessor{cdrFormat: CSV, cdrSourceIds: []string{"TEST_CDRC"}, duMultiplyFactors: []float64{0}, cdrFields: [][]*config.CfgCdrField{cdrFields}}
+	cdrRow := []string{"*data", "1"}
+	rtCdr, err := csvProcessor.recordToStoredCdr(cdrRow, 0)
+	if err != nil {
+		t.Error("Failed to parse CDR in rated cdr", err)
+	}
+	var sTime time.Time
+	expectedCdr := &engine.StoredCdr{
+		CgrId:       utils.Sha1("", sTime.String()),
+		TOR:         cdrRow[0],
+		CdrHost:     "0.0.0.0",
+		CdrSource:   "TEST_CDRC",
+		Usage:       time.Duration(1) * time.Second,
+		ExtraFields: map[string]string{},
+		Cost:        -1,
+	}
+	if !reflect.DeepEqual(expectedCdr, rtCdr) {
+		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
+	}
+	csvProcessor.duMultiplyFactors = []float64{1024}
+	expectedCdr = &engine.StoredCdr{
+		CgrId:       utils.Sha1("", sTime.String()),
+		TOR:         cdrRow[0],
+		CdrHost:     "0.0.0.0",
+		CdrSource:   "TEST_CDRC",
+		Usage:       time.Duration(1024) * time.Second,
+		ExtraFields: map[string]string{},
+		Cost:        -1,
+	}
+	if rtCdr, _ := csvProcessor.recordToStoredCdr(cdrRow, 0); !reflect.DeepEqual(expectedCdr, rtCdr) {
+		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
+	}
+	cdrRow = []string{"*voice", "1"}
+	expectedCdr = &engine.StoredCdr{
+		CgrId:       utils.Sha1("", sTime.String()),
+		TOR:         cdrRow[0],
+		CdrHost:     "0.0.0.0",
+		CdrSource:   "TEST_CDRC",
+		Usage:       time.Duration(1) * time.Second,
+		ExtraFields: map[string]string{},
+		Cost:        -1,
+	}
+	if rtCdr, _ := csvProcessor.recordToStoredCdr(cdrRow, 0); !reflect.DeepEqual(expectedCdr, rtCdr) {
+		t.Errorf("Expected: \n%v, \nreceived: \n%v", expectedCdr, rtCdr)
+	}
+}
+
+func TestCsvPairToRecord(t *testing.T) {
+	eRecord := []string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454408", "*prepaid", "1001", "1002", "", "3401:2069362475", "2"}
+	invPr := &PartialFlatstoreRecord{Method: "INVITE", Timestamp: time.Date(2015, 7, 9, 15, 6, 48, 0, time.UTC),
+		Values: []string{"INVITE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454408", "*prepaid", "1001", "1002", "", "3401:2069362475"}}
+	byePr := &PartialFlatstoreRecord{Method: "BYE", Timestamp: time.Date(2015, 7, 9, 15, 6, 50, 0, time.UTC),
+		Values: []string{"BYE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454410", "", "", "", "", "3401:2069362475"}}
+	if rec, err := pairToRecord(invPr, byePr); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eRecord, rec) {
+		t.Errorf("Expected: %+v, received: %+v", eRecord, rec)
+	}
+	if rec, err := pairToRecord(byePr, invPr); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eRecord, rec) {
+		t.Errorf("Expected: %+v, received: %+v", eRecord, rec)
+	}
+	if _, err := pairToRecord(byePr, byePr); err == nil || err.Error() != "MISSING_INVITE" {
+		t.Error(err)
+	}
+	if _, err := pairToRecord(invPr, invPr); err == nil || err.Error() != "MISSING_BYE" {
+		t.Error(err)
+	}
+	byePr.Values = []string{"BYE", "2daec40c", "548625ac", "dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0", "200", "OK", "1436454410", "", "", "", "3401:2069362475"} // Took one value out
+	if _, err := pairToRecord(invPr, byePr); err == nil || err.Error() != "INCONSISTENT_VALUES_LENGTH" {
+		t.Error(err)
+	}
+}

cdrc/flatstore_local_test.go

@@ -0,0 +1,161 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"io/ioutil"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+)
+
+var flatstoreCfgPath string
+var flatstoreCfg *config.CGRConfig
+var flatstoreRpc *rpc.Client
+var flatstoreCdrcCfg *config.CdrcConfig
+
+var fullSuccessfull = `INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475
+BYE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454410|||||3401:2069362475
+INVITE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454647|*postpaid|1002|1001||1877:893549741
+BYE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4cca@0:0:0:0:0:0:0:0|200|OK|1436454651|||||1877:893549741
+INVITE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454657|*prepaid|1001|1002||2407:1884881533
+BYE|36e39a5|42d996f9|3a63321dd3b325eec688dc2aefb6ac2d@0:0:0:0:0:0:0:0|200|OK|1436454661|||||2407:1884881533
+INVITE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454690|*prepaid|1001|1002||3099:1909036290
+BYE|3111f3c9|49ca4c42|a58ebaae40d08d6757d8424fb09c4c54@0:0:0:0:0:0:0:0|200|OK|1436454692|||||3099:1909036290
+`
+
+var fullMissed = `INVITE|ef6c6256|da501581|0bfdd176d1b93e7df3de5c6f4873ee04@0:0:0:0:0:0:0:0|487|Request Terminated|1436454643|*prepaid|1001|1002||1224:339382783
+INVITE|7905e511||81880da80a94bda81b425b09009e055c@0:0:0:0:0:0:0:0|404|Not Found|1436454668|*prepaid|1001|1002||1980:1216490844
+INVITE|324cb497|d4af7023|8deaadf2ae9a17809a391f05af31afb0@0:0:0:0:0:0:0:0|486|Busy here|1436454687|*postpaid|1002|1001||474:130115066`
+
+var part1 = `BYE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4ccb@0:0:0:0:0:0:0:0|200|OK|1436454651|||||1877:893549742
+`
+
+var part2 = `INVITE|f9d3d5c3|c863a6e3|214d8f52b566e33a9349b184e72a4ccb@0:0:0:0:0:0:0:0|200|OK|1436454647|*postpaid|1002|1003||1877:893549742
+INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475`
+
+func TestFlatstoreLclInitCfg(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	flatstoreCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcflatstore")
+	if flatstoreCfg, err = config.NewCGRConfigFromFolder(flatstoreCfgPath); err != nil {
+		t.Fatal("Got config error: ", err.Error())
+	}
+}
+
+// InitDb so we can rely on count
+func TestFlatstoreLclInitCdrDb(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := engine.InitStorDb(flatstoreCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Creates cdr files and moves them into processing folder
+func TestFlatstoreLclCreateCdrFiles(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if flatstoreCfg == nil {
+		t.Fatal("Empty default cdrc configuration")
+	}
+	flatstoreCdrcCfg = flatstoreCfg.CdrcProfiles["/tmp/cgr_flatstore/cdrc/in"]["FLATSTORE"]
+	if err := os.RemoveAll(flatstoreCdrcCfg.CdrInDir); err != nil {
+		t.Fatal("Error removing folder: ", flatstoreCdrcCfg.CdrInDir, err)
+	}
+	if err := os.MkdirAll(flatstoreCdrcCfg.CdrInDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", flatstoreCdrcCfg.CdrInDir, err)
+	}
+	if err := os.RemoveAll(flatstoreCdrcCfg.CdrOutDir); err != nil {
+		t.Fatal("Error removing folder: ", flatstoreCdrcCfg.CdrOutDir, err)
+	}
+	if err := os.MkdirAll(flatstoreCdrcCfg.CdrOutDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", flatstoreCdrcCfg.CdrOutDir, err)
+	}
+}
+
+func TestFlatstoreLclStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if _, err := engine.StopStartEngine(flatstoreCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestFlatstoreLclRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	flatstoreRpc, err = jsonrpc.Dial("tcp", flatstoreCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+func TestFlatstoreLclProcessFiles(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if err := ioutil.WriteFile(path.Join("/tmp", "acc_1.log"), []byte(fullSuccessfull), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	if err := ioutil.WriteFile(path.Join("/tmp", "missed_calls_1.log"), []byte(fullMissed), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	if err := ioutil.WriteFile(path.Join("/tmp", "acc_2.log"), []byte(part1), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	if err := ioutil.WriteFile(path.Join("/tmp", "acc_3.log"), []byte(part2), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	//Rename(oldpath, newpath string)
+	for _, fileName := range []string{"acc_1.log", "missed_calls_1.log", "acc_2.log", "acc_3.log"} {
+		if err := os.Rename(path.Join("/tmp", fileName), path.Join(flatstoreCdrcCfg.CdrInDir, fileName)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	time.Sleep(time.Duration(2) * time.Second) // Give time for processing to happen and the .unparired file to be written
+	filesInDir, _ := ioutil.ReadDir(flatstoreCdrcCfg.CdrInDir)
+	if len(filesInDir) != 0 {
+		t.Errorf("Files in cdrcInDir: %+v", filesInDir)
+	}
+	filesOutDir, _ := ioutil.ReadDir(flatstoreCdrcCfg.CdrOutDir)
+	if len(filesOutDir) != 5 {
+		t.Errorf("In CdrcOutDir, expecting 5 files, got: %d", len(filesOutDir))
+	}
+	ePartContent := "INVITE|2daec40c|548625ac|dd0c4c617a9919d29a6175cdff223a9e@0:0:0:0:0:0:0:0|200|OK|1436454408|*prepaid|1001|1002||3401:2069362475\n"
+	if partContent, err := ioutil.ReadFile(path.Join(flatstoreCdrcCfg.CdrOutDir, "acc_3.log.unpaired")); err != nil {
+		t.Error(err)
+	} else if ePartContent != string(partContent) {
+		t.Errorf("Expecting: %s, received: %s", ePartContent, string(partContent))
+	}
+}

cdrc/fwv.go

@@ -0,0 +1,246 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func fwvValue(cdrLine string, indexStart, width int, padding string) string {
+	rawVal := cdrLine[indexStart : indexStart+width]
+	switch padding {
+	case "left":
+		rawVal = strings.TrimLeft(rawVal, " ")
+	case "right":
+		rawVal = strings.TrimRight(rawVal, " ")
+	case "zeroleft":
+		rawVal = strings.TrimLeft(rawVal, "0 ")
+	case "zeroright":
+		rawVal = strings.TrimRight(rawVal, "0 ")
+	}
+	return rawVal
+}
+
+func NewFwvRecordsProcessor(file *os.File, cdrcCfgs map[string]*config.CdrcConfig, dfltCfg *config.CdrcConfig, httpClient *http.Client, httpSkipTlsCheck bool, timezone string) *FwvRecordsProcessor {
+	return &FwvRecordsProcessor{file: file, cdrcCfgs: cdrcCfgs, dfltCfg: dfltCfg, httpSkipTlsCheck: httpSkipTlsCheck, timezone: timezone}
+}
+
+type FwvRecordsProcessor struct {
+	file             *os.File
+	cdrcCfgs         map[string]*config.CdrcConfig
+	dfltCfg          *config.CdrcConfig // General parameters
+	httpClient       *http.Client
+	httpSkipTlsCheck bool
+	timezone         string
+	lineLen          int64             // Length of the line in the file
+	offset           int64             // Index of the next byte to process
+	trailerOffset    int64             // Index where trailer starts, to be used as boundary when reading cdrs
+	headerCdr        *engine.StoredCdr // Cache here the general purpose stored CDR
+}
+
+// Sets the line length based on first line, sets offset back to initial after reading
+func (self *FwvRecordsProcessor) setLineLen() error {
+	rdr := bufio.NewReader(self.file)
+	readBytes, err := rdr.ReadBytes('\n')
+	if err != nil {
+		return err
+	}
+	self.lineLen = int64(len(readBytes))
+	if _, err := self.file.Seek(0, 0); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (self *FwvRecordsProcessor) ProcessNextRecord() ([]*engine.StoredCdr, error) {
+	defer func() { self.offset += self.lineLen }() // Schedule increasing the offset once we are out from processing the record
+	if self.offset == 0 {                          // First time, set the necessary offsets
+		if err := self.setLineLen(); err != nil {
+			engine.Logger.Err(fmt.Sprintf("<Cdrc> Row 0, error: cannot set lineLen: %s", err.Error()))
+			return nil, io.EOF
+		}
+		if len(self.dfltCfg.TrailerFields) != 0 {
+			if fi, err := self.file.Stat(); err != nil {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Row 0, error: cannot get file stats: %s", err.Error()))
+				return nil, err
+			} else {
+				self.trailerOffset = fi.Size() - self.lineLen
+			}
+		}
+		if len(self.dfltCfg.HeaderFields) != 0 { // ToDo: Process here the header fields
+			if err := self.processHeader(); err != nil {
+				engine.Logger.Err(fmt.Sprintf("<Cdrc> Row 0, error reading header: %s", err.Error()))
+				return nil, io.EOF
+			}
+			return nil, nil
+		}
+	}
+	recordCdrs := make([]*engine.StoredCdr, 0) // More CDRs based on the number of filters and field templates
+	if self.trailerOffset != 0 && self.offset >= self.trailerOffset {
+		if err := self.processTrailer(); err != nil && err != io.EOF {
+			engine.Logger.Err(fmt.Sprintf("<Cdrc> Read trailer error: %s ", err.Error()))
+		}
+		return nil, io.EOF
+	}
+	buf := make([]byte, self.lineLen)
+	nRead, err := self.file.Read(buf)
+	if err != nil {
+		return nil, err
+	} else if nRead != len(buf) {
+		engine.Logger.Err(fmt.Sprintf("<Cdrc> Could not read complete line, have instead: %s", string(buf)))
+		return nil, io.EOF
+	}
+	record := string(buf)
+	for cfgKey := range self.cdrcCfgs {
+		if passes := self.recordPassesCfgFilter(record, cfgKey); !passes {
+			continue
+		}
+		if storedCdr, err := self.recordToStoredCdr(record, cfgKey); err != nil {
+			return nil, fmt.Errorf("Failed converting to StoredCdr, error: %s", err.Error())
+		} else {
+			recordCdrs = append(recordCdrs, storedCdr)
+		}
+	}
+	return recordCdrs, nil
+}
+
+func (self *FwvRecordsProcessor) recordPassesCfgFilter(record, configKey string) bool {
+	filterPasses := true
+	for _, rsrFilter := range self.cdrcCfgs[configKey].CdrFilter {
+		if rsrFilter == nil { // Nil filter does not need to match anything
+			continue
+		}
+		if cfgFieldIdx, _ := strconv.Atoi(rsrFilter.Id); len(record) <= cfgFieldIdx {
+			fmt.Errorf("Ignoring record: %v - cannot compile filter %+v", record, rsrFilter)
+			return false
+		} else if !rsrFilter.FilterPasses(record[cfgFieldIdx:]) {
+			filterPasses = false
+			break
+		}
+	}
+	return filterPasses
+}
+
+// Converts a record (header or normal) to StoredCdr
+func (self *FwvRecordsProcessor) recordToStoredCdr(record string, cfgKey string) (*engine.StoredCdr, error) {
+	var err error
+	var lazyHttpFields []*config.CfgCdrField
+	var cfgFields []*config.CfgCdrField
+	var duMultiplyFactor float64
+	var storedCdr *engine.StoredCdr
+	if self.headerCdr != nil { // Clone the header CDR so we can use it as base to future processing (inherit fields defined there)
+		storedCdr = self.headerCdr.Clone()
+	} else {
+		storedCdr = &engine.StoredCdr{CdrHost: "0.0.0.0", ExtraFields: make(map[string]string), Cost: -1}
+	}
+	if cfgKey == "*header" {
+		cfgFields = self.dfltCfg.HeaderFields
+		storedCdr.CdrSource = self.dfltCfg.CdrSourceId
+		duMultiplyFactor = self.dfltCfg.DataUsageMultiplyFactor
+	} else {
+		cfgFields = self.cdrcCfgs[cfgKey].ContentFields
+		storedCdr.CdrSource = self.cdrcCfgs[cfgKey].CdrSourceId
+		duMultiplyFactor = self.cdrcCfgs[cfgKey].DataUsageMultiplyFactor
+	}
+	for _, cdrFldCfg := range cfgFields {
+		var fieldVal string
+		switch cdrFldCfg.Type {
+		case utils.CDRFIELD:
+			for _, cfgFieldRSR := range cdrFldCfg.Value {
+				if cfgFieldRSR.IsStatic() {
+					fieldVal += cfgFieldRSR.ParseValue("")
+				} else { // Dynamic value extracted using index
+					if cfgFieldIdx, _ := strconv.Atoi(cfgFieldRSR.Id); len(record) <= cfgFieldIdx {
+						return nil, fmt.Errorf("Ignoring record: %v - cannot extract field %s", record, cdrFldCfg.Tag)
+					} else {
+						fieldVal += cfgFieldRSR.ParseValue(fwvValue(record, cfgFieldIdx, cdrFldCfg.Width, cdrFldCfg.Padding))
+					}
+				}
+			}
+		case utils.HTTP_POST:
+			lazyHttpFields = append(lazyHttpFields, cdrFldCfg) // Will process later so we can send an estimation of storedCdr to http server
+		default:
+			//return nil, fmt.Errorf("Unsupported field type: %s", cdrFldCfg.Type)
+			continue // Don't do anything for unsupported fields
+		}
+		if err := populateStoredCdrField(storedCdr, cdrFldCfg.CdrFieldId, fieldVal, self.timezone); err != nil {
+			return nil, err
+		}
+	}
+	if storedCdr.CgrId == "" && storedCdr.AccId != "" && cfgKey != "*header" {
+		storedCdr.CgrId = utils.Sha1(storedCdr.AccId, storedCdr.SetupTime.UTC().String())
+	}
+	if storedCdr.TOR == utils.DATA && duMultiplyFactor != 0 {
+		storedCdr.Usage = time.Duration(float64(storedCdr.Usage.Nanoseconds()) * duMultiplyFactor)
+	}
+	for _, httpFieldCfg := range lazyHttpFields { // Lazy process the http fields
+		var outValByte []byte
+		var fieldVal, httpAddr string
+		for _, rsrFld := range httpFieldCfg.Value {
+			httpAddr += rsrFld.ParseValue("")
+		}
+		if outValByte, err = utils.HttpJsonPost(httpAddr, self.httpSkipTlsCheck, storedCdr); err != nil && httpFieldCfg.Mandatory {
+			return nil, err
+		} else {
+			fieldVal = string(outValByte)
+			if len(fieldVal) == 0 && httpFieldCfg.Mandatory {
+				return nil, fmt.Errorf("MandatoryIeMissing: Empty result for http_post field: %s", httpFieldCfg.Tag)
+			}
+			if err := populateStoredCdrField(storedCdr, httpFieldCfg.CdrFieldId, fieldVal, self.timezone); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return storedCdr, nil
+}
+
+func (self *FwvRecordsProcessor) processHeader() error {
+	buf := make([]byte, self.lineLen)
+	if nRead, err := self.file.Read(buf); err != nil {
+		return err
+	} else if nRead != len(buf) {
+		return fmt.Errorf("In header, line len: %d, have read: %d", self.lineLen, nRead)
+	}
+	var err error
+	if self.headerCdr, err = self.recordToStoredCdr(string(buf), "*header"); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (self *FwvRecordsProcessor) processTrailer() error {
+	buf := make([]byte, self.lineLen)
+	if nRead, err := self.file.ReadAt(buf, self.trailerOffset); err != nil {
+		return err
+	} else if nRead != len(buf) {
+		return fmt.Errorf("In trailer, line len: %d, have read: %d", self.lineLen, nRead)
+	}
+	//engine.Logger.Debug(fmt.Sprintf("Have read trailer: <%q>", string(buf)))
+	return nil
+}

cdrc/fwv_local_test.go

@@ -0,0 +1,150 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"io/ioutil"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"os"
+	"path"
+	"testing"
+	"time"
+)
+
+var fwvCfgPath string
+var fwvCfg *config.CGRConfig
+var fwvRpc *rpc.Client
+var fwvCdrcCfg *config.CdrcConfig
+
+var FW_CDR_FILE1 = `HDR0001DDB     ABC                                     Some Connect A.B.                       DDB-Some-10022-20120711-309.CDR         00030920120711100255                                                                    
+CDR0000010  0 20120708181506000123451234         0040123123120                  004                                            000018009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009
+CDR0000020  0 20120708190945000123451234         0040123123120                  004                                            000016009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009
+CDR0000030  0 20120708191009000123451234         0040123123120                  004                                            000020009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009
+CDR0000040  0 20120708231043000123451234         0040123123120                  004                                            000011009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009
+CDR0000050  0 20120709122216000123451235         004212                         004                                            000217009980010001ISDN  ABC   10Buiten uw regio                         HMR 00000000190000000000
+CDR0000060  0 20120709130542000123451236         0012323453                     004                                            000019009980010001ISDN  ABC   35Sterdiensten                            AP  00000000190000000000
+CDR0000070  0 20120709140032000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000080  0 20120709140142000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000090  0 20120709150305000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000100  0 20120709150414000123451237         0040012323453100               001                                            000057009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000110  0 20120709150531000123451237         0040012323453100               001                                            000059009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000120  0 20120709150635000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000130  0 20120709151756000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000140  0 20120709154549000123451237         0040012323453100               001                                            000052009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000150  0 20120709154701000123451237         0040012323453100               001                                            000121009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000160  0 20120709154842000123451237         0040012323453100               001                                            000055009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000170  0 20120709154956000123451237         0040012323453100               001                                            000115009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000180  0 20120709155131000123451237         0040012323453100               001                                            000059009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000190  0 20120709155236000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000200  0 20120709160309000123451237         0040012323453100               001                                            000100009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000210  0 20120709160415000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000220  0 20120709161739000123451237         0040012323453100               001                                            000058009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000230  0 20120709170356000123123459         0040123234531                  004                                            000012002760010001ISDN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000240  0 20120709181036000123123450         0012323453                     004                                            000042009980010001ISDN  ABC   05Binnen uw regio                         AP  00000010190000000010
+CDR0000250  0 20120709191245000123123458         0040123232350                  004                                            000012002760000001PSTN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000260  0 20120709202324000123123459         0040123234531                  004                                            000011002760010001ISDN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000270  0 20120709211756000123451237         0040012323453100               001                                            000051009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000280  0 20120709211852000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000290  0 20120709212904000123123458         0040123232350                  004                                            000012002760000001PSTN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000300  0 20120709073707000123123459         0040123234531                  004                                            000012002760010001ISDN  276   10Buiten uw regio                         TB  00000009190000000009
+CDR0000310  0 20120709085451000123451237         0040012323453100               001                                            000744009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000320  0 20120709091756000123451237         0040012323453100               001                                            000050009980030001ISDN  ABD   20Internationaal                          NLB 00000000190000000000
+CDR0000330  0 20120710070434000123123458         0040123232350                  004                                            000012002760000001PSTN  276   10Buiten uw regio                         TB  00000009190000000009
+TRL0001DDB     ABC                                     Some Connect A.B.                       DDB-Some-10022-20120711-309.CDR         0003090000003300000030550000000001000000000100Y                                         
+`
+
+func TestFwvLclInitCfg(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	fwvCfgPath = path.Join(*dataDir, "conf", "samples", "cdrcfwv")
+	if fwvCfg, err = config.NewCGRConfigFromFolder(fwvCfgPath); err != nil {
+		t.Fatal("Got config error: ", err.Error())
+	}
+}
+
+// Creates cdr files and moves them into processing folder
+func TestFwvLclCreateCdrFiles(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if fwvCfg == nil {
+		t.Fatal("Empty default cdrc configuration")
+	}
+	fwvCdrcCfg = fwvCfg.CdrcProfiles["/tmp/cgr_fwv/cdrc/in"]["FWV1"]
+	if err := os.RemoveAll(fwvCdrcCfg.CdrInDir); err != nil {
+		t.Fatal("Error removing folder: ", fwvCdrcCfg.CdrInDir, err)
+	}
+	if err := os.MkdirAll(fwvCdrcCfg.CdrInDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", fwvCdrcCfg.CdrInDir, err)
+	}
+	if err := os.RemoveAll(fwvCdrcCfg.CdrOutDir); err != nil {
+		t.Fatal("Error removing folder: ", fwvCdrcCfg.CdrOutDir, err)
+	}
+	if err := os.MkdirAll(fwvCdrcCfg.CdrOutDir, 0755); err != nil {
+		t.Fatal("Error creating folder: ", fwvCdrcCfg.CdrOutDir, err)
+	}
+}
+
+func TestFwvLclStartEngine(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if _, err := engine.StopStartEngine(fwvCfgPath, *waitRater); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Connect rpc client to rater
+func TestFwvLclRpcConn(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	fwvRpc, err = jsonrpc.Dial("tcp", fwvCfg.RPCJSONListen) // We connect over JSON so we can also troubleshoot if needed
+	if err != nil {
+		t.Fatal("Could not connect to rater: ", err.Error())
+	}
+}
+
+func TestFwvLclProcessFiles(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	fileName := "test1.fwv"
+	if err := ioutil.WriteFile(path.Join("/tmp", fileName), []byte(FW_CDR_FILE1), 0644); err != nil {
+		t.Fatal(err.Error)
+	}
+	if err := os.Rename(path.Join("/tmp", fileName), path.Join(fwvCdrcCfg.CdrInDir, fileName)); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Duration(1) * time.Second)
+	filesInDir, _ := ioutil.ReadDir(fwvCdrcCfg.CdrInDir)
+	if len(filesInDir) != 0 {
+		t.Errorf("Files in cdrcInDir: %d", len(filesInDir))
+	}
+	filesOutDir, _ := ioutil.ReadDir(fwvCdrcCfg.CdrOutDir)
+	if len(filesOutDir) != 1 {
+		t.Errorf("In CdrcOutDir, expecting 1 files, got: %d", len(filesOutDir))
+	}
+}

cdrc/fwv_test.go

@@ -0,0 +1,55 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdrc
+
+import (
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/utils"
+	"testing"
+)
+
+func TestFwvValue(t *testing.T) {
+	cdrLine := "CDR0000010  0 20120708181506000123451234         0040123123120                  004                                            000018009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009"
+	if val := fwvValue(cdrLine, 30, 19, "right"); val != "0123451234" {
+		t.Errorf("Received: <%s>", val)
+	}
+	if val := fwvValue(cdrLine, 14, 16, "right"); val != "2012070818150600" { // SetupTime
+		t.Errorf("Received: <%s>", val)
+	}
+	if val := fwvValue(cdrLine, 127, 8, "right"); val != "00001800" { // Usage
+		t.Errorf("Received: <%s>", val)
+	}
+	cdrLine = "HDR0001DDB     ABC                                     Some Connect A.B.                       DDB-Some-10022-20120711-309.CDR         00030920120711100255                                                "
+	if val := fwvValue(cdrLine, 135, 6, "zeroleft"); val != "309" {
+		t.Errorf("Received: <%s>", val)
+	}
+
+}
+
+func TestFwvRecordPassesCfgFilter(t *testing.T) {
+	//record, configKey string) bool {
+	cgrConfig, _ := config.NewDefaultCGRConfig()
+	cdrcConfig := cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"][utils.META_DEFAULT] // We don't really care that is for .csv since all we want to test are the filters
+	cdrcConfig.CdrFilter = utils.ParseRSRFieldsMustCompile(`~52:s/^0(\d{9})/+49${1}/(+49123123120)`, utils.INFIELD_SEP)
+	fwvRp := &FwvRecordsProcessor{cdrcCfgs: cgrConfig.CdrcProfiles["/var/log/cgrates/cdrc/in"]}
+	cdrLine := "CDR0000010  0 20120708181506000123451234         0040123123120                  004                                            000018009980010001ISDN  ABC   10Buiten uw regio                         EHV 00000009190000000009"
+	if passesFilter := fwvRp.recordPassesCfgFilter(cdrLine, utils.META_DEFAULT); !passesFilter {
+		t.Error("Not passes filter")
+	}
+}

cdre/cdrexporter.go

@@ -0,0 +1,534 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdre
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+const (
+	COST_DETAILS         = "cost_details"
+	DATETIME             = "datetime"
+	META_EXPORTID        = "*export_id"
+	META_TIMENOW         = "*time_now"
+	META_FIRSTCDRATIME   = "*first_cdr_atime"
+	META_LASTCDRATIME    = "*last_cdr_atime"
+	META_NRCDRS          = "*cdrs_number"
+	META_DURCDRS         = "*cdrs_duration"
+	META_SMSUSAGE        = "*sms_usage"
+	META_GENERICUSAGE    = "*generic_usage"
+	META_DATAUSAGE       = "*data_usage"
+	META_COSTCDRS        = "*cdrs_cost"
+	META_MASKDESTINATION = "*mask_destination"
+	META_FORMATCOST      = "*format_cost"
+)
+
+var err error
+
+func NewCdrExporter(cdrs []*engine.StoredCdr, cdrDb engine.CdrStorage, exportTpl *config.CdreConfig, cdrFormat string, fieldSeparator rune, exportId string,
+	dataUsageMultiplyFactor, smsUsageMultiplyFactor, genericUsageMultiplyFactor, costMultiplyFactor float64,
+	costShiftDigits, roundDecimals, cgrPrecision int, maskDestId string, maskLen int, httpSkipTlsCheck bool, timezone string) (*CdrExporter, error) {
+	if len(cdrs) == 0 { // Nothing to export
+		return nil, nil
+	}
+	cdre := &CdrExporter{
+		cdrs:                    cdrs,
+		cdrDb:                   cdrDb,
+		exportTemplate:          exportTpl,
+		cdrFormat:               cdrFormat,
+		fieldSeparator:          fieldSeparator,
+		exportId:                exportId,
+		dataUsageMultiplyFactor: dataUsageMultiplyFactor,
+		smsUsageMultiplyFactor:  smsUsageMultiplyFactor,
+		costMultiplyFactor:      costMultiplyFactor,
+		costShiftDigits:         costShiftDigits,
+		roundDecimals:           roundDecimals,
+		cgrPrecision:            cgrPrecision,
+		maskDestId:              maskDestId,
+		httpSkipTlsCheck:        httpSkipTlsCheck,
+		timezone:                timezone,
+		maskLen:                 maskLen,
+		negativeExports:         make(map[string]string),
+	}
+	if err := cdre.processCdrs(); err != nil {
+		return nil, err
+	}
+	return cdre, nil
+}
+
+type CdrExporter struct {
+	cdrs           []*engine.StoredCdr
+	cdrDb          engine.CdrStorage // Used to extract cost_details if these are requested
+	exportTemplate *config.CdreConfig
+	cdrFormat      string // csv, fwv
+	fieldSeparator rune
+	exportId       string // Unique identifier or this export
+	dataUsageMultiplyFactor,
+	smsUsageMultiplyFactor, // Multiply the SMS usage (eg: some billing systems billing them as minutes)
+	genericUsageMultiplyFactor,
+	costMultiplyFactor float64
+	costShiftDigits, roundDecimals, cgrPrecision                    int
+	maskDestId                                                      string
+	maskLen                                                         int
+	httpSkipTlsCheck                                                bool
+	timezone                                                        string
+	header, trailer                                                 []string   // Header and Trailer fields
+	content                                                         [][]string // Rows of cdr fields
+	firstCdrATime, lastCdrATime                                     time.Time
+	numberOfRecords                                                 int
+	totalDuration, totalDataUsage, totalSmsUsage, totalGenericUsage time.Duration
+
+	totalCost                       float64
+	firstExpOrderId, lastExpOrderId int64
+	positiveExports                 []string          // CGRIds of successfully exported CDRs
+	negativeExports                 map[string]string // CgrIds of failed exports
+}
+
+// Return Json marshaled callCost attached to
+// Keep it separately so we test only this part in local tests
+func (cdre *CdrExporter) getCdrCostDetails(cgrId, runId string) (string, error) {
+	cc, err := cdre.cdrDb.GetCallCostLog(cgrId, "", runId)
+	if err != nil {
+		return "", err
+	} else if cc == nil {
+		return "", nil
+	}
+	ccJson, _ := json.Marshal(cc)
+	return string(ccJson), nil
+}
+
+func (cdre *CdrExporter) getCombimedCdrFieldVal(processedCdr *engine.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) {
+	var combinedVal string // Will result as combination of the field values, filters must match
+	for _, filterRule := range cfgCdrFld.FieldFilter {
+		fltrPass, ftrPassValue := processedCdr.PassesFieldFilter(filterRule)
+		if !fltrPass {
+			return "", nil
+		}
+		for _, cdr := range cdre.cdrs {
+			if cdr.CgrId != processedCdr.CgrId {
+				continue // We only care about cdrs with same primary cdr behind
+			}
+			if cdr.FieldAsString(&utils.RSRField{Id: filterRule.Id}) == ftrPassValue { // First CDR with filte
+				for _, rsrRule := range cfgCdrFld.Value {
+					combinedVal += cdr.FieldAsString(rsrRule)
+				}
+			}
+		}
+	}
+	return combinedVal, nil
+}
+
+// Check if the destination should be masked in output
+func (cdre *CdrExporter) maskedDestination(destination string) bool {
+	if len(cdre.maskDestId) != 0 && engine.CachedDestHasPrefix(cdre.maskDestId, destination) {
+		return true
+	}
+	return false
+}
+
+func (cdre *CdrExporter) getDateTimeFieldVal(cdr *engine.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) {
+	if len(cfgCdrFld.Value) == 0 {
+		return "", nil
+	}
+	for _, fltrRl := range cfgCdrFld.FieldFilter {
+		if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass {
+			return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl)
+		}
+	}
+	layout := cfgCdrFld.Layout
+	if len(layout) == 0 {
+		layout = time.RFC3339
+	}
+	if dtFld, err := utils.ParseTimeDetectLayout(cdr.FieldAsString(cfgCdrFld.Value[0]), cdre.timezone); err != nil { // Only one rule makes sense here
+		return "", err
+	} else {
+		return dtFld.Format(layout), nil
+	}
+}
+
+// Extracts the value specified by cfgHdr out of cdr
+func (cdre *CdrExporter) cdrFieldValue(cdr *engine.StoredCdr, cfgCdrFld *config.CfgCdrField) (string, error) {
+	for _, fltrRl := range cfgCdrFld.FieldFilter {
+		if fltrPass, _ := cdr.PassesFieldFilter(fltrRl); !fltrPass {
+			return "", fmt.Errorf("Field: %s not matching filter rule %v", fltrRl.Id, fltrRl)
+		}
+	}
+	layout := cfgCdrFld.Layout
+	if len(layout) == 0 {
+		layout = time.RFC3339
+	}
+	var retVal string // Concatenate the resulting values
+	for _, rsrFld := range cfgCdrFld.Value {
+		var cdrVal string
+		switch rsrFld.Id {
+		case COST_DETAILS: // Special case when we need to further extract cost_details out of logDb
+			if cdr.ExtraFields[COST_DETAILS], err = cdre.getCdrCostDetails(cdr.CgrId, cdr.MediationRunId); err != nil {
+				return "", err
+			} else {
+				cdrVal = cdr.FieldAsString(rsrFld)
+			}
+		case utils.COST:
+			cdrVal = cdr.FormatCost(cdre.costShiftDigits, cdre.roundDecimals)
+		case utils.USAGE:
+			cdrVal = cdr.FormatUsage(layout)
+		case utils.SETUP_TIME:
+			cdrVal = cdr.SetupTime.Format(layout)
+		case utils.ANSWER_TIME: // Format time based on layout
+			cdrVal = cdr.AnswerTime.Format(layout)
+		case utils.DESTINATION:
+			cdrVal = cdr.FieldAsString(rsrFld)
+			if cdre.maskLen != -1 && cdre.maskedDestination(cdrVal) {
+				cdrVal = MaskDestination(cdrVal, cdre.maskLen)
+			}
+		default:
+			cdrVal = cdr.FieldAsString(rsrFld)
+		}
+		retVal += cdrVal
+	}
+	return retVal, nil
+}
+
+// Handle various meta functions used in header/trailer
+func (cdre *CdrExporter) metaHandler(tag, arg string) (string, error) {
+	switch tag {
+	case META_EXPORTID:
+		return cdre.exportId, nil
+	case META_TIMENOW:
+		return time.Now().Format(arg), nil
+	case META_FIRSTCDRATIME:
+		return cdre.firstCdrATime.Format(arg), nil
+	case META_LASTCDRATIME:
+		return cdre.lastCdrATime.Format(arg), nil
+	case META_NRCDRS:
+		return strconv.Itoa(cdre.numberOfRecords), nil
+	case META_DURCDRS:
+		emulatedCdr := &engine.StoredCdr{TOR: utils.VOICE, Usage: cdre.totalDuration}
+		return emulatedCdr.FormatUsage(arg), nil
+	case META_SMSUSAGE:
+		emulatedCdr := &engine.StoredCdr{TOR: utils.SMS, Usage: cdre.totalSmsUsage}
+		return emulatedCdr.FormatUsage(arg), nil
+	case META_GENERICUSAGE:
+		emulatedCdr := &engine.StoredCdr{TOR: utils.GENERIC, Usage: cdre.totalGenericUsage}
+		return emulatedCdr.FormatUsage(arg), nil
+	case META_DATAUSAGE:
+		emulatedCdr := &engine.StoredCdr{TOR: utils.DATA, Usage: cdre.totalDataUsage}
+		return emulatedCdr.FormatUsage(arg), nil
+	case META_COSTCDRS:
+		return strconv.FormatFloat(utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE), 'f', -1, 64), nil
+	case META_MASKDESTINATION:
+		if cdre.maskedDestination(arg) {
+			return "1", nil
+		}
+		return "0", nil
+	default:
+		return "", fmt.Errorf("Unsupported METATAG: %s", tag)
+	}
+}
+
+// Compose and cache the header
+func (cdre *CdrExporter) composeHeader() error {
+	for _, cfgFld := range cdre.exportTemplate.HeaderFields {
+		var outVal string
+		switch cfgFld.Type {
+		case utils.FILLER:
+			outVal = cfgFld.Value.Id()
+			cfgFld.Padding = "right"
+		case utils.CONSTANT:
+			outVal = cfgFld.Value.Id()
+		case utils.METATAG:
+			outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cfgFld.Layout)
+		default:
+			return fmt.Errorf("Unsupported field type: %s", cfgFld.Type)
+		}
+		if err != nil {
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR header, field %s, error: %s", cfgFld.Tag, err.Error()))
+			return err
+		}
+		fmtOut := outVal
+		if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR header, field %s, error: %s", cfgFld.Tag, err.Error()))
+			return err
+		}
+		cdre.header = append(cdre.header, fmtOut)
+	}
+	return nil
+}
+
+// Compose and cache the trailer
+func (cdre *CdrExporter) composeTrailer() error {
+	for _, cfgFld := range cdre.exportTemplate.TrailerFields {
+		var outVal string
+		switch cfgFld.Type {
+		case utils.FILLER:
+			outVal = cfgFld.Value.Id()
+			cfgFld.Padding = "right"
+		case utils.CONSTANT:
+			outVal = cfgFld.Value.Id()
+		case utils.METATAG:
+			outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cfgFld.Layout)
+		default:
+			return fmt.Errorf("Unsupported field type: %s", cfgFld.Type)
+		}
+		if err != nil {
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, field: %s, error: %s", cfgFld.Tag, err.Error()))
+			return err
+		}
+		fmtOut := outVal
+		if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR trailer, field: %s, error: %s", cfgFld.Tag, err.Error()))
+			return err
+		}
+		cdre.trailer = append(cdre.trailer, fmtOut)
+	}
+	return nil
+}
+
+// Write individual cdr into content buffer, build stats
+func (cdre *CdrExporter) processCdr(cdr *engine.StoredCdr) error {
+	if cdr == nil || len(cdr.CgrId) == 0 { // We do not export empty CDRs
+		return nil
+	} else if cdr.ExtraFields == nil { // Avoid assignment in nil map if not initialized
+		cdr.ExtraFields = make(map[string]string)
+	}
+	// Cost multiply
+	if cdre.dataUsageMultiplyFactor != 0.0 && cdr.TOR == utils.DATA {
+		cdr.UsageMultiply(cdre.dataUsageMultiplyFactor, cdre.cgrPrecision)
+	} else if cdre.smsUsageMultiplyFactor != 0 && cdr.TOR == utils.SMS {
+		cdr.UsageMultiply(cdre.smsUsageMultiplyFactor, cdre.cgrPrecision)
+	} else if cdre.genericUsageMultiplyFactor != 0 && cdr.TOR == utils.GENERIC {
+		cdr.UsageMultiply(cdre.genericUsageMultiplyFactor, cdre.cgrPrecision)
+	}
+	if cdre.costMultiplyFactor != 0.0 {
+		cdr.CostMultiply(cdre.costMultiplyFactor, cdre.cgrPrecision)
+	}
+	var err error
+	cdrRow := make([]string, len(cdre.exportTemplate.ContentFields))
+	for idx, cfgFld := range cdre.exportTemplate.ContentFields {
+		var outVal string
+		switch cfgFld.Type {
+		case utils.FILLER:
+			outVal = cfgFld.Value.Id()
+			cfgFld.Padding = "right"
+		case utils.CONSTANT:
+			outVal = cfgFld.Value.Id()
+		case utils.CDRFIELD:
+			outVal, err = cdre.cdrFieldValue(cdr, cfgFld)
+		case DATETIME:
+			outVal, err = cdre.getDateTimeFieldVal(cdr, cfgFld)
+		case utils.HTTP_POST:
+			var outValByte []byte
+			httpAddr := cfgFld.Value.Id()
+			if len(httpAddr) == 0 {
+				err = fmt.Errorf("Empty http address for field %s type %s", cfgFld.Tag, cfgFld.Type)
+			} else if outValByte, err = utils.HttpJsonPost(httpAddr, cdre.httpSkipTlsCheck, cdr); err == nil {
+				outVal = string(outValByte)
+				if len(outVal) == 0 && cfgFld.Mandatory {
+					err = fmt.Errorf("Empty result for http_post field: %s", cfgFld.Tag)
+				}
+			}
+		case utils.COMBIMED:
+			outVal, err = cdre.getCombimedCdrFieldVal(cdr, cfgFld)
+		case utils.METATAG:
+			if cfgFld.Value.Id() == META_MASKDESTINATION {
+				outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cdr.FieldAsString(&utils.RSRField{Id: utils.DESTINATION}))
+			} else {
+				outVal, err = cdre.metaHandler(cfgFld.Value.Id(), cfgFld.Layout)
+			}
+		}
+		if err != nil {
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s and runid: %s, error: %s", cdr.CgrId, cdr.MediationRunId, err.Error()))
+			return err
+		}
+		fmtOut := outVal
+		if fmtOut, err = FmtFieldWidth(outVal, cfgFld.Width, cfgFld.Strip, cfgFld.Padding, cfgFld.Mandatory); err != nil {
+			engine.Logger.Err(fmt.Sprintf("<CdreFw> Cannot export CDR with cgrid: %s, runid: %s, fieldName: %s, fieldValue: %s, error: %s", cdr.CgrId, cdr.MediationRunId, cfgFld.Tag, outVal, err.Error()))
+			return err
+		}
+		cdrRow[idx] += fmtOut
+	}
+	if len(cdrRow) == 0 { // No CDR data, most likely no configuration fields defined
+		return nil
+	} else {
+		cdre.content = append(cdre.content, cdrRow)
+	}
+	// Done with writing content, compute stats here
+	if cdre.firstCdrATime.IsZero() || cdr.AnswerTime.Before(cdre.firstCdrATime) {
+		cdre.firstCdrATime = cdr.AnswerTime
+	}
+	if cdr.AnswerTime.After(cdre.lastCdrATime) {
+		cdre.lastCdrATime = cdr.AnswerTime
+	}
+	cdre.numberOfRecords += 1
+	if cdr.TOR == utils.VOICE { // Only count duration for non data cdrs
+		cdre.totalDuration += cdr.Usage
+	}
+	if cdr.TOR == utils.SMS { // Count usage for SMS
+		cdre.totalSmsUsage += cdr.Usage
+	}
+	if cdr.TOR == utils.GENERIC { // Count usage for GENERIC
+		cdre.totalGenericUsage += cdr.Usage
+	}
+	if cdr.TOR == utils.DATA { // Count usage for DATA
+		cdre.totalDataUsage += cdr.Usage
+	}
+	if cdr.Cost != -1 {
+		cdre.totalCost += cdr.Cost
+		cdre.totalCost = utils.Round(cdre.totalCost, cdre.roundDecimals, utils.ROUNDING_MIDDLE)
+	}
+	if cdre.firstExpOrderId > cdr.OrderId || cdre.firstExpOrderId == 0 {
+		cdre.firstExpOrderId = cdr.OrderId
+	}
+	if cdre.lastExpOrderId < cdr.OrderId {
+		cdre.lastExpOrderId = cdr.OrderId
+	}
+	return nil
+}
+
+// Builds header, content and trailers
+func (cdre *CdrExporter) processCdrs() error {
+	for _, cdr := range cdre.cdrs {
+		if err := cdre.processCdr(cdr); err != nil {
+			cdre.negativeExports[cdr.CgrId] = err.Error()
+		} else {
+			cdre.positiveExports = append(cdre.positiveExports, cdr.CgrId)
+		}
+	}
+	// Process header and trailer after processing cdrs since the metatag functions can access stats out of built cdrs
+	if cdre.exportTemplate.HeaderFields != nil {
+		if err := cdre.composeHeader(); err != nil {
+			return err
+		}
+	}
+	if cdre.exportTemplate.TrailerFields != nil {
+		if err := cdre.composeTrailer(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Simple write method
+func (cdre *CdrExporter) writeOut(ioWriter io.Writer) error {
+	if len(cdre.header) != 0 {
+		for _, fld := range append(cdre.header, "\n") {
+			if _, err := io.WriteString(ioWriter, fld); err != nil {
+				return err
+			}
+		}
+	}
+	for _, cdrContent := range cdre.content {
+		for _, cdrFld := range append(cdrContent, "\n") {
+			if _, err := io.WriteString(ioWriter, cdrFld); err != nil {
+				return err
+			}
+		}
+	}
+	if len(cdre.trailer) != 0 {
+		for _, fld := range append(cdre.trailer, "\n") {
+			if _, err := io.WriteString(ioWriter, fld); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// csvWriter specific method
+func (cdre *CdrExporter) writeCsv(csvWriter *csv.Writer) error {
+	csvWriter.Comma = cdre.fieldSeparator
+	if len(cdre.header) != 0 {
+		if err := csvWriter.Write(cdre.header); err != nil {
+			return err
+		}
+	}
+	for _, cdrContent := range cdre.content {
+		if err := csvWriter.Write(cdrContent); err != nil {
+			return err
+		}
+	}
+	if len(cdre.trailer) != 0 {
+		if err := csvWriter.Write(cdre.trailer); err != nil {
+			return err
+		}
+	}
+	csvWriter.Flush()
+	return nil
+}
+
+// General method to write the content out to a file
+func (cdre *CdrExporter) WriteToFile(filePath string) error {
+	fileOut, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	defer fileOut.Close()
+	switch cdre.cdrFormat {
+	case utils.DRYRUN:
+		return nil
+	case utils.CDRE_FIXED_WIDTH:
+		if err := cdre.writeOut(fileOut); err != nil {
+			return utils.NewErrServerError(err)
+		}
+	case utils.CSV:
+		csvWriter := csv.NewWriter(fileOut)
+		if err := cdre.writeCsv(csvWriter); err != nil {
+			return utils.NewErrServerError(err)
+		}
+	}
+	return nil
+}
+
+// Return the first exported Cdr OrderId
+func (cdre *CdrExporter) FirstOrderId() int64 {
+	return cdre.firstExpOrderId
+}
+
+// Return the last exported Cdr OrderId
+func (cdre *CdrExporter) LastOrderId() int64 {
+	return cdre.lastExpOrderId
+}
+
+// Return total cost in the exported cdrs
+func (cdre *CdrExporter) TotalCost() float64 {
+	return cdre.totalCost
+}
+
+func (cdre *CdrExporter) TotalExportedCdrs() int {
+	return cdre.numberOfRecords
+}
+
+// Return successfully exported CgrIds
+func (cdre *CdrExporter) PositiveExports() []string {
+	return cdre.positiveExports
+}
+
+// Return failed exported CgrIds together with the reason
+func (cdre *CdrExporter) NegativeExports() map[string]string {
+	return cdre.negativeExports
+}

cdre/cdrexporter_test.go

@@ -0,0 +1,124 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdre
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func TestCdreGetCombimedCdrFieldVal(t *testing.T) {
+	cfg, _ := config.NewDefaultCGRConfig()
+	cdrs := []*engine.StoredCdr{
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+			Usage: time.Duration(10) * time.Second, MediationRunId: "RUN_RTL", Cost: 1.01},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf2", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+			Usage: time.Duration(10) * time.Second, MediationRunId: "CUSTOMER1", Cost: 2.01},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+			Usage: time.Duration(10) * time.Second, MediationRunId: "CUSTOMER1", Cost: 3.01},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+			Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 4.01},
+		&engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+			ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+			Category: "call", Account: "1000", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+			Usage: time.Duration(10) * time.Second, MediationRunId: "RETAIL1", Cost: 5.01},
+	}
+	cdre, err := NewCdrExporter(cdrs, nil, cfg.CdreProfiles["*default"], cfg.CdreProfiles["*default"].CdrFormat, cfg.CdreProfiles["*default"].FieldSeparator,
+		"firstexport", 0.0, 0.0, 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", 0, cfg.HttpSkipTlsVerify, "")
+	if err != nil {
+		t.Error("Unexpected error received: ", err)
+	}
+	fltrRule, _ := utils.ParseRSRFields("~mediation_runid:s/default/RUN_RTL/", utils.INFIELD_SEP)
+	val, _ := utils.ParseRSRFields("cost", utils.INFIELD_SEP)
+	cfgCdrFld := &config.CfgCdrField{Tag: "cost", Type: "cdrfield", CdrFieldId: "cost", Value: val, FieldFilter: fltrRule}
+	if costVal, err := cdre.getCombimedCdrFieldVal(cdrs[3], cfgCdrFld); err != nil {
+		t.Error(err)
+	} else if costVal != "1.01" {
+		t.Error("Expecting: 1.01, received: ", costVal)
+	}
+	fltrRule, _ = utils.ParseRSRFields("~mediation_runid:s/default/RETAIL1/", utils.INFIELD_SEP)
+	val, _ = utils.ParseRSRFields("account", utils.INFIELD_SEP)
+	cfgCdrFld = &config.CfgCdrField{Tag: "account", Type: "cdrfield", CdrFieldId: "account", Value: val, FieldFilter: fltrRule}
+	if acntVal, err := cdre.getCombimedCdrFieldVal(cdrs[3], cfgCdrFld); err != nil {
+		t.Error(err)
+	} else if acntVal != "1000" {
+		t.Error("Expecting: 1000, received: ", acntVal)
+	}
+}
+
+func TestGetDateTimeFieldVal(t *testing.T) {
+	cdreTst := new(CdrExporter)
+	cdrTst := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+		Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.01,
+		ExtraFields: map[string]string{"stop_time": "2014-06-11 19:19:00 +0000 UTC", "fieldextr2": "valextr2"}}
+	val, _ := utils.ParseRSRFields("stop_time", utils.INFIELD_SEP)
+	layout := "2006-01-02 15:04:05"
+	cfgCdrFld := &config.CfgCdrField{Tag: "stop_time", Type: "cdrfield", CdrFieldId: "stop_time", Value: val, Layout: layout}
+	if cdrVal, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err != nil {
+		t.Error(err)
+	} else if cdrVal != "2014-06-11 19:19:00" {
+		t.Error("Expecting: 2014-06-11 19:19:00, got: ", cdrVal)
+	}
+	// Test filter
+	fltr, _ := utils.ParseRSRFields("~tenant:s/(.+)/itsyscom.com/", utils.INFIELD_SEP)
+	cfgCdrFld = &config.CfgCdrField{Tag: "stop_time", Type: "cdrfield", CdrFieldId: "stop_time", Value: val, FieldFilter: fltr, Layout: layout}
+	if _, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err == nil {
+		t.Error(err)
+	}
+	val, _ = utils.ParseRSRFields("fieldextr2", utils.INFIELD_SEP)
+	cfgCdrFld = &config.CfgCdrField{Tag: "stop_time", Type: "cdrfield", CdrFieldId: "stop_time", Value: val, Layout: layout}
+	// Test time parse error
+	if _, err := cdreTst.getDateTimeFieldVal(cdrTst, cfgCdrFld); err == nil {
+		t.Error("Should give error here, got none.")
+	}
+}
+
+func TestCdreCdrFieldValue(t *testing.T) {
+	cdre := new(CdrExporter)
+	cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+		Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.01}
+	val, _ := utils.ParseRSRFields("destination", utils.INFIELD_SEP)
+	cfgCdrFld := &config.CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: val}
+	if val, err := cdre.cdrFieldValue(cdr, cfgCdrFld); err != nil {
+		t.Error(err)
+	} else if val != cdr.Destination {
+		t.Errorf("Expecting: %s, received: %s", cdr.Destination, val)
+	}
+	fltr, _ := utils.ParseRSRFields("~tenant:s/(.+)/itsyscom.com/", utils.INFIELD_SEP)
+	cfgCdrFld = &config.CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: val, FieldFilter: fltr}
+	if _, err := cdre.cdrFieldValue(cdr, cfgCdrFld); err == nil {
+		t.Error("Failed to use filter")
+	}
+}

cdre/csv_test.go

@@ -0,0 +1,87 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdre
+
+import (
+	"bytes"
+	"encoding/csv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func TestCsvCdrWriter(t *testing.T) {
+	writer := &bytes.Buffer{}
+	cfg, _ := config.NewDefaultCGRConfig()
+	storedCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+		Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID,
+		ExtraFields: map[string]string{"extra1": "val_extra1", "extra2": "val_extra2", "extra3": "val_extra3"}, Cost: 1.01,
+	}
+	cdre, err := NewCdrExporter([]*engine.StoredCdr{storedCdr1}, nil, cfg.CdreProfiles["*default"], utils.CSV, ',', "firstexport", 0.0, 0.0, 0.0, 0.0, 0, 4,
+		cfg.RoundingDecimals, "", 0, cfg.HttpSkipTlsVerify, "")
+	if err != nil {
+		t.Error("Unexpected error received: ", err)
+	}
+	csvWriter := csv.NewWriter(writer)
+	if err := cdre.writeCsv(csvWriter); err != nil {
+		t.Error("Unexpected error: ", err)
+	}
+	expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6,*default,*voice,dsafdsaf,*rated,*out,cgrates.org,call,1001,1001,1002,2013-11-07T08:42:25Z,2013-11-07T08:42:26Z,10,1.0100`
+	result := strings.TrimSpace(writer.String())
+	if result != expected {
+		t.Errorf("Expected: \n%s received: \n%s.", expected, result)
+	}
+	if cdre.TotalCost() != 1.01 {
+		t.Error("Unexpected TotalCost: ", cdre.TotalCost())
+	}
+}
+
+func TestAlternativeFieldSeparator(t *testing.T) {
+	writer := &bytes.Buffer{}
+	cfg, _ := config.NewDefaultCGRConfig()
+	storedCdr1 := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Unix(1383813745, 0).UTC().String()), TOR: utils.VOICE, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1001", Subject: "1001", Destination: "1002", SetupTime: time.Unix(1383813745, 0).UTC(), AnswerTime: time.Unix(1383813746, 0).UTC(),
+		Usage: time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID,
+		ExtraFields: map[string]string{"extra1": "val_extra1", "extra2": "val_extra2", "extra3": "val_extra3"}, Cost: 1.01,
+	}
+	cdre, err := NewCdrExporter([]*engine.StoredCdr{storedCdr1}, nil, cfg.CdreProfiles["*default"], utils.CSV, '|',
+		"firstexport", 0.0, 0.0, 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", 0, cfg.HttpSkipTlsVerify, "")
+	if err != nil {
+		t.Error("Unexpected error received: ", err)
+	}
+	csvWriter := csv.NewWriter(writer)
+	if err := cdre.writeCsv(csvWriter); err != nil {
+		t.Error("Unexpected error: ", err)
+	}
+	expected := `dbafe9c8614c785a65aabd116dd3959c3c56f7f6|*default|*voice|dsafdsaf|*rated|*out|cgrates.org|call|1001|1001|1002|2013-11-07T08:42:25Z|2013-11-07T08:42:26Z|10|1.0100`
+	result := strings.TrimSpace(writer.String())
+	if result != expected {
+		t.Errorf("Expected: \n%s received: \n%s.", expected, result)
+	}
+	if cdre.TotalCost() != 1.01 {
+		t.Error("Unexpected TotalCost: ", cdre.TotalCost())
+	}
+}

cdre/fixedwidth_test.go

@@ -0,0 +1,241 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdre
+
+import (
+	"bytes"
+	"math"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var hdrJsnCfgFlds = []*config.CdrFieldJsonCfg{
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TypeOfRecord"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("10"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler1"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(3)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("DistributorCode"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("VOI"), Width: utils.IntPointer(3)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FileSeqNr"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_EXPORTID),
+		Width: utils.IntPointer(5), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("zeroleft")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("LastCdr"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_LASTCDRATIME),
+		Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FileCreationfTime"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_TIMENOW),
+		Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FileVersion"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("01"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler2"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(105)},
+}
+
+var contentJsnCfgFlds = []*config.CdrFieldJsonCfg{
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TypeOfRecord"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("20"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Account"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.ACCOUNT), Width: utils.IntPointer(12),
+		Strip: utils.StringPointer("left"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Subject"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.SUBJECT), Width: utils.IntPointer(5),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("CLI"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("cli"), Width: utils.IntPointer(15),
+		Strip: utils.StringPointer("xright"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Destination"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.DESTINATION), Width: utils.IntPointer(24),
+		Strip: utils.StringPointer("xright"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TOR"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("02"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("SubtypeTOR"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("11"), Width: utils.IntPointer(4),
+		Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("SetupTime"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.SETUP_TIME), Width: utils.IntPointer(12),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right"), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Duration"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.USAGE), Width: utils.IntPointer(6),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right"), Layout: utils.StringPointer(utils.SECONDS)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("DataVolume"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(6)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TaxCode"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("1"), Width: utils.IntPointer(1)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("OperatorCode"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("opercode"), Width: utils.IntPointer(2),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("ProductId"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("productid"), Width: utils.IntPointer(5),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("NetworkId"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("3"), Width: utils.IntPointer(1)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("CallId"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.ACCID), Width: utils.IntPointer(16),
+		Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(8)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(8)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TerminationCode"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer("operator;product"),
+		Width: utils.IntPointer(5), Strip: utils.StringPointer("right"), Padding: utils.StringPointer("right")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Cost"), Type: utils.StringPointer(utils.CDRFIELD), Value: utils.StringPointer(utils.COST), Width: utils.IntPointer(9),
+		Padding: utils.StringPointer("zeroleft")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("DestinationPrivacy"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_MASKDESTINATION),
+		Width: utils.IntPointer(1)},
+}
+
+var trailerJsnCfgFlds = []*config.CdrFieldJsonCfg{
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("TypeOfRecord"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("90"), Width: utils.IntPointer(2)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler1"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(3)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("DistributorCode"), Type: utils.StringPointer(utils.CONSTANT), Value: utils.StringPointer("VOI"), Width: utils.IntPointer(3)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FileSeqNr"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_EXPORTID), Width: utils.IntPointer(5),
+		Strip: utils.StringPointer("right"), Padding: utils.StringPointer("zeroleft")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("NumberOfRecords"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_NRCDRS),
+		Width: utils.IntPointer(6), Padding: utils.StringPointer("zeroleft")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("CdrsDuration"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_DURCDRS),
+		Width: utils.IntPointer(8), Padding: utils.StringPointer("zeroleft"), Layout: utils.StringPointer(utils.SECONDS)},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("FirstCdrTime"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_FIRSTCDRATIME),
+		Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("LastCdrTime"), Type: utils.StringPointer(utils.METATAG), Value: utils.StringPointer(META_LASTCDRATIME),
+		Width: utils.IntPointer(12), Layout: utils.StringPointer("020106150400")},
+	&config.CdrFieldJsonCfg{Tag: utils.StringPointer("Filler2"), Type: utils.StringPointer(utils.FILLER), Width: utils.IntPointer(93)},
+}
+
+var hdrCfgFlds, contentCfgFlds, trailerCfgFlds []*config.CfgCdrField
+
+// Write one CDR and test it's results only for content buffer
+func TestWriteCdr(t *testing.T) {
+	var err error
+	wrBuf := &bytes.Buffer{}
+	cfg, _ := config.NewDefaultCGRConfig()
+	if hdrCfgFlds, err = config.CfgCdrFieldsFromCdrFieldsJsonCfg(hdrJsnCfgFlds); err != nil {
+		t.Error(err)
+	}
+	if contentCfgFlds, err = config.CfgCdrFieldsFromCdrFieldsJsonCfg(contentJsnCfgFlds); err != nil {
+		t.Error(err)
+	}
+	if trailerCfgFlds, err = config.CfgCdrFieldsFromCdrFieldsJsonCfg(trailerJsnCfgFlds); err != nil {
+		t.Error(err)
+	}
+	cdreCfg := &config.CdreConfig{
+		CdrFormat:     utils.CDRE_FIXED_WIDTH,
+		HeaderFields:  hdrCfgFlds,
+		ContentFields: contentCfgFlds,
+		TrailerFields: trailerCfgFlds,
+	}
+	cdr := &engine.StoredCdr{CgrId: utils.Sha1("dsafdsaf", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()),
+		TOR: utils.VOICE, OrderId: 1, AccId: "dsafdsaf", CdrHost: "192.168.1.1",
+		ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1001", Subject: "1001", Destination: "1002",
+		SetupTime:  time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC),
+		AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC),
+		Usage:      time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.34567,
+		ExtraFields: map[string]string{"field_extr1": "val_extr1", "fieldextr2": "valextr2"},
+	}
+	cdre, err := NewCdrExporter([]*engine.StoredCdr{cdr}, nil, cdreCfg, utils.CDRE_FIXED_WIDTH, ',', "fwv_1", 0.0, 0.0, 0.0, 0.0, 0, 4,
+		cfg.RoundingDecimals, "", -1, cfg.HttpSkipTlsVerify, "")
+	if err != nil {
+		t.Error(err)
+	}
+	eHeader := "10   VOIfwv_107111308420018011511340001                                                                                                         \n"
+	eContentOut := "201001        1001                1002                    0211  07111308420010          1       3dsafdsaf                             0002.34570\n"
+	eTrailer := "90   VOIfwv_100000100000010071113084200071113084200                                                                                             \n"
+	if err := cdre.writeOut(wrBuf); err != nil {
+		t.Error(err)
+	}
+	allOut := wrBuf.String()
+	eAllOut := eHeader + eContentOut + eTrailer
+	if math.Mod(float64(len(allOut)), 145) != 0 {
+		t.Error("Unexpected export content length", len(allOut))
+	} else if len(allOut) != len(eAllOut) {
+		t.Errorf("Output does not match expected length. Have output %q, expecting: %q", allOut, eAllOut)
+	}
+	// Test stats
+	if !cdre.firstCdrATime.Equal(cdr.AnswerTime) {
+		t.Error("Unexpected firstCdrATime in stats: ", cdre.firstCdrATime)
+	} else if !cdre.lastCdrATime.Equal(cdr.AnswerTime) {
+		t.Error("Unexpected lastCdrATime in stats: ", cdre.lastCdrATime)
+	} else if cdre.numberOfRecords != 1 {
+		t.Error("Unexpected number of records in the stats: ", cdre.numberOfRecords)
+	} else if cdre.totalDuration != cdr.Usage {
+		t.Error("Unexpected total duration in the stats: ", cdre.totalDuration)
+	} else if cdre.totalCost != utils.Round(cdr.Cost, cdre.roundDecimals, utils.ROUNDING_MIDDLE) {
+		t.Error("Unexpected total cost in the stats: ", cdre.totalCost)
+	}
+	if cdre.FirstOrderId() != 1 {
+		t.Error("Unexpected FirstOrderId", cdre.FirstOrderId())
+	}
+	if cdre.LastOrderId() != 1 {
+		t.Error("Unexpected LastOrderId", cdre.LastOrderId())
+	}
+	if cdre.TotalCost() != utils.Round(cdr.Cost, cdre.roundDecimals, utils.ROUNDING_MIDDLE) {
+		t.Error("Unexpected TotalCost: ", cdre.TotalCost())
+	}
+}
+
+func TestWriteCdrs(t *testing.T) {
+	wrBuf := &bytes.Buffer{}
+	cdreCfg := &config.CdreConfig{
+		CdrFormat:     utils.CDRE_FIXED_WIDTH,
+		HeaderFields:  hdrCfgFlds,
+		ContentFields: contentCfgFlds,
+		TrailerFields: trailerCfgFlds,
+	}
+	cdr1 := &engine.StoredCdr{CgrId: utils.Sha1("aaa1", time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC).String()),
+		TOR: utils.VOICE, OrderId: 2, AccId: "aaa1", CdrHost: "192.168.1.1", ReqType: utils.META_RATED, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1001", Subject: "1001", Destination: "1010",
+		SetupTime:  time.Date(2013, 11, 7, 8, 42, 20, 0, time.UTC),
+		AnswerTime: time.Date(2013, 11, 7, 8, 42, 26, 0, time.UTC),
+		Usage:      time.Duration(10) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.25,
+		ExtraFields: map[string]string{"productnumber": "12341", "fieldextr2": "valextr2"},
+	}
+	cdr2 := &engine.StoredCdr{CgrId: utils.Sha1("aaa2", time.Date(2013, 11, 7, 7, 42, 20, 0, time.UTC).String()),
+		TOR: utils.VOICE, OrderId: 4, AccId: "aaa2", CdrHost: "192.168.1.2", ReqType: utils.META_PREPAID, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1002", Subject: "1002", Destination: "1011",
+		SetupTime:  time.Date(2013, 11, 7, 7, 42, 20, 0, time.UTC),
+		AnswerTime: time.Date(2013, 11, 7, 7, 42, 26, 0, time.UTC),
+		Usage:      time.Duration(5) * time.Minute, MediationRunId: utils.DEFAULT_RUNID, Cost: 1.40001,
+		ExtraFields: map[string]string{"productnumber": "12342", "fieldextr2": "valextr2"},
+	}
+	cdr3 := &engine.StoredCdr{}
+	cdr4 := &engine.StoredCdr{CgrId: utils.Sha1("aaa3", time.Date(2013, 11, 7, 9, 42, 18, 0, time.UTC).String()),
+		TOR: utils.VOICE, OrderId: 3, AccId: "aaa4", CdrHost: "192.168.1.4", ReqType: utils.META_POSTPAID, Direction: "*out", Tenant: "cgrates.org",
+		Category: "call", Account: "1004", Subject: "1004", Destination: "1013",
+		SetupTime:  time.Date(2013, 11, 7, 9, 42, 18, 0, time.UTC),
+		AnswerTime: time.Date(2013, 11, 7, 9, 42, 26, 0, time.UTC),
+		Usage:      time.Duration(20) * time.Second, MediationRunId: utils.DEFAULT_RUNID, Cost: 2.34567,
+		ExtraFields: map[string]string{"productnumber": "12344", "fieldextr2": "valextr2"},
+	}
+	cfg, _ := config.NewDefaultCGRConfig()
+	cdre, err := NewCdrExporter([]*engine.StoredCdr{cdr1, cdr2, cdr3, cdr4}, nil, cdreCfg, utils.CDRE_FIXED_WIDTH, ',',
+		"fwv_1", 0.0, 0.0, 0.0, 0.0, 0, 4, cfg.RoundingDecimals, "", -1, cfg.HttpSkipTlsVerify, "")
+	if err != nil {
+		t.Error(err)
+	}
+	if err := cdre.writeOut(wrBuf); err != nil {
+		t.Error(err)
+	}
+	if len(wrBuf.String()) != 725 {
+		t.Error("Output buffer does not contain expected info. Expecting len: 725, got: ", len(wrBuf.String()))
+	}
+	// Test stats
+	if !cdre.firstCdrATime.Equal(cdr2.AnswerTime) {
+		t.Error("Unexpected firstCdrATime in stats: ", cdre.firstCdrATime)
+	}
+	if !cdre.lastCdrATime.Equal(cdr4.AnswerTime) {
+		t.Error("Unexpected lastCdrATime in stats: ", cdre.lastCdrATime)
+	}
+	if cdre.numberOfRecords != 3 {
+		t.Error("Unexpected number of records in the stats: ", cdre.numberOfRecords)
+	}
+	if cdre.totalDuration != time.Duration(330)*time.Second {
+		t.Error("Unexpected total duration in the stats: ", cdre.totalDuration)
+	}
+	if cdre.totalCost != 5.9957 {
+		t.Error("Unexpected total cost in the stats: ", cdre.totalCost)
+	}
+	if cdre.FirstOrderId() != 2 {
+		t.Error("Unexpected FirstOrderId", cdre.FirstOrderId())
+	}
+	if cdre.LastOrderId() != 4 {
+		t.Error("Unexpected LastOrderId", cdre.LastOrderId())
+	}
+	if cdre.TotalCost() != 5.9957 {
+		t.Error("Unexpected TotalCost: ", cdre.TotalCost())
+	}
+}

cdre/libcdre.go

@@ -0,0 +1,93 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdre
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+// Used as generic function logic for various fields
+
+// Attributes
+//  source - the base source
+//  width - the field width
+//  strip - if present it will specify the strip strategy, when missing strip will not be allowed
+//  padding - if present it will specify the padding strategy to use, left, right, zeroleft, zeroright
+func FmtFieldWidth(source string, width int, strip, padding string, mandatory bool) (string, error) {
+	if mandatory && len(source) == 0 {
+		return "", errors.New("Empty source value")
+	}
+	if width == 0 { // Disable width processing if not defined
+		return source, nil
+	}
+	if len(source) == width { // the source is exactly the maximum length
+		return source, nil
+	}
+	if len(source) > width { //the source is bigger than allowed
+		if len(strip) == 0 {
+			return "", fmt.Errorf("Source %s is bigger than the width %d, no strip defied", source, width)
+		}
+		if strip == "right" {
+			return source[:width], nil
+		} else if strip == "xright" {
+			return source[:width-1] + "x", nil // Suffix with x to mark prefix
+		} else if strip == "left" {
+			diffIndx := len(source) - width
+			return source[diffIndx:], nil
+		} else if strip == "xleft" { // Prefix one x to mark stripping
+			diffIndx := len(source) - width
+			return "x" + source[diffIndx+1:], nil
+		}
+	} else { //the source is smaller as the maximum allowed
+		if len(padding) == 0 {
+			return "", fmt.Errorf("Source %s is smaller than the width %d, no padding defined", source, width)
+		}
+		var paddingFmt string
+		switch padding {
+		case "right":
+			paddingFmt = fmt.Sprintf("%%-%ds", width)
+		case "left":
+			paddingFmt = fmt.Sprintf("%%%ds", width)
+		case "zeroleft":
+			paddingFmt = fmt.Sprintf("%%0%ds", width)
+		}
+		if len(paddingFmt) != 0 {
+			return fmt.Sprintf(paddingFmt, source), nil
+		}
+	}
+	return source, nil
+}
+
+// Mask a number of characters in the suffix of the destination
+func MaskDestination(dest string, maskLen int) string {
+	destLen := len(dest)
+	if maskLen < 0 {
+		return dest
+	} else if maskLen > destLen {
+		maskLen = destLen
+	}
+	dest = dest[:destLen-maskLen]
+	for i := 0; i < maskLen; i++ {
+		dest += utils.MASK_CHAR
+	}
+	return dest
+}

cdre/libcdre_test.go

@@ -0,0 +1,133 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package cdre
+
+import (
+	"testing"
+)
+
+func TestMandatory(t *testing.T) {
+	_, err := FmtFieldWidth("", 0, "", "", true)
+	if err == nil {
+		t.Errorf("Failed to detect mandatory value")
+	}
+}
+
+func TestMaxLen(t *testing.T) {
+	result, err := FmtFieldWidth("test", 4, "", "", false)
+	expected := "test"
+	if err != nil || result != expected {
+		t.Errorf("Expected \"test\" was \"%s\"", result)
+	}
+}
+
+func TestRPadding(t *testing.T) {
+	result, err := FmtFieldWidth("test", 8, "", "right", false)
+	expected := "test    "
+	if err != nil || result != expected {
+		t.Errorf("Expected \"%s \" was \"%s\"", expected, result)
+	}
+}
+
+func TestPaddingFiller(t *testing.T) {
+	result, err := FmtFieldWidth("", 8, "", "right", false)
+	expected := "        "
+	if err != nil || result != expected {
+		t.Errorf("Expected \"%s \" was \"%s\"", expected, result)
+	}
+}
+
+func TestLPadding(t *testing.T) {
+	result, err := FmtFieldWidth("test", 8, "", "left", false)
+	expected := "    test"
+	if err != nil || result != expected {
+		t.Errorf("Expected \"%s \" was \"%s\"", expected, result)
+	}
+}
+
+func TestZeroLPadding(t *testing.T) {
+	result, err := FmtFieldWidth("test", 8, "", "zeroleft", false)
+	expected := "0000test"
+	if err != nil || result != expected {
+		t.Errorf("Expected \"%s \" was \"%s\"", expected, result)
+	}
+}
+
+func TestRStrip(t *testing.T) {
+	result, err := FmtFieldWidth("test", 2, "right", "", false)
+	expected := "te"
+	if err != nil || result != expected {
+		t.Errorf("Expected \"%s \" was \"%s\"", expected, result)
+	}
+}
+
+func TestXRStrip(t *testing.T) {
+	result, err := FmtFieldWidth("test", 3, "xright", "", false)
+	expected := "tex"
+	if err != nil || result != expected {
+		t.Errorf("Expected \"%s \" was \"%s\"", expected, result)
+	}
+}
+
+func TestLStrip(t *testing.T) {
+	result, err := FmtFieldWidth("test", 2, "left", "", false)
+	expected := "st"
+	if err != nil || result != expected {
+		t.Errorf("Expected \"%s \" was \"%s\"", expected, result)
+	}
+}
+
+func TestXLStrip(t *testing.T) {
+	result, err := FmtFieldWidth("test", 3, "xleft", "", false)
+	expected := "xst"
+	if err != nil || result != expected {
+		t.Errorf("Expected \"%s \" was \"%s\"", expected, result)
+	}
+}
+
+func TestStripNotAllowed(t *testing.T) {
+	_, err := FmtFieldWidth("test", 3, "", "", false)
+	if err == nil {
+		t.Error("Expected error")
+	}
+}
+
+func TestPaddingNotAllowed(t *testing.T) {
+	_, err := FmtFieldWidth("test", 5, "", "", false)
+	if err == nil {
+		t.Error("Expected error")
+	}
+}
+
+func TestMaskDestination(t *testing.T) {
+	dest := "+4986517174963"
+	if destMasked := MaskDestination(dest, 3); destMasked != "+4986517174***" {
+		t.Error("Unexpected mask applied", destMasked)
+	}
+	if destMasked := MaskDestination(dest, -1); destMasked != dest {
+		t.Error("Negative maskLen should not modify destination", destMasked)
+	}
+	if destMasked := MaskDestination(dest, 0); destMasked != dest {
+		t.Error("Zero maskLen should not modify destination", destMasked)
+	}
+	if destMasked := MaskDestination(dest, 100); destMasked != "**************" {
+		t.Error("High maskLen should return complete mask", destMasked)
+	}
+
+}

cmd/cgr-console/cgr-console.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -19,49 +19,164 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package main
 
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
-	"github.com/cgrates/cgrates/console"
-	"github.com/cgrates/cgrates/rater"
+	"io"
 	"log"
-	"net/rpc"
-	"net/rpc/jsonrpc"
 	"os"
+	"strings"
+
+	"github.com/cgrates/cgrates/console"
+	"github.com/cgrates/cgrates/utils"
+	"github.com/cgrates/rpcclient"
+	"github.com/peterh/liner"
 )
 
 var (
+	history_fn   = os.Getenv("HOME") + "/.cgr_history"
 	version      = flag.Bool("version", false, "Prints the application version.")
+	verbose      = flag.Bool("verbose", false, "Show extra info about command execution.")
 	server       = flag.String("server", "127.0.0.1:2012", "server address host:port")
-	rpc_encoding = flag.String("rpc_encoding", "gob", "RPC encoding used <gob|json>")
+	rpc_encoding = flag.String("rpc_encoding", "json", "RPC encoding used <gob|json>")
+	client       *rpcclient.RpcClient
 )
 
+func executeCommand(command string) {
+	if strings.TrimSpace(command) == "" {
+		return
+	}
+	if strings.TrimSpace(command) == "help" {
+		commands := console.GetCommands()
+		fmt.Println("Commands:")
+		for name, cmd := range commands {
+			fmt.Print(name, cmd.Usage())
+		}
+		return
+	}
+	if strings.HasPrefix(command, "help") {
+		words := strings.Split(command, " ")
+		if len(words) > 1 {
+			commands := console.GetCommands()
+			if cmd, ok := commands[words[1]]; ok {
+				fmt.Print(cmd.Usage())
+			} else {
+				fmt.Print("Available commands: ")
+				for name, _ := range commands {
+					fmt.Print(name + " ")
+				}
+				fmt.Println()
+			}
+			return
+		}
+	}
+	cmd, cmdErr := console.GetCommandValue(command, *verbose)
+	if cmdErr != nil {
+		fmt.Println(cmdErr)
+		return
+	}
+	if cmd.RpcMethod() != "" {
+		res := cmd.RpcResult()
+		param := cmd.RpcParams(false)
+		//log.Print(reflect.TypeOf(param))
+		switch param.(type) {
+		case *console.EmptyWrapper:
+			param = ""
+		case *console.StringWrapper:
+			param = param.(*console.StringWrapper).Item
+		case *console.StringSliceWrapper:
+			param = param.(*console.StringSliceWrapper).Items
+		case *console.StringMapWrapper:
+			param = param.(*console.StringMapWrapper).Items
+		}
+		//log.Printf("Param: %+v", param)
+
+		if rpcErr := client.Call(cmd.RpcMethod(), param, res); rpcErr != nil {
+			fmt.Println("Error executing command: " + rpcErr.Error())
+		} else {
+			result, _ := json.MarshalIndent(res, "", " ")
+			fmt.Println(string(result))
+		}
+	} else {
+		fmt.Println(cmd.LocalExecute())
+	}
+}
+
 func main() {
 	flag.Parse()
 	if *version {
-		fmt.Println("CGRateS " + rater.VERSION)
+		fmt.Println("CGRateS " + utils.VERSION)
 		return
 	}
-	var client *rpc.Client
 	var err error
-	if *rpc_encoding == "json" {
-		client, err = jsonrpc.Dial("tcp", *server)
-	} else {
-		client, err = rpc.Dial("tcp", *server)
-	}
+	client, err = rpcclient.NewRpcClient("tcp", *server, 3, 3, *rpc_encoding)
 	if err != nil {
 		flag.PrintDefaults()
 		log.Fatal("Could not connect to server " + *server)
 	}
-	defer client.Close()
-	// Strict command parsing starts here
-	args := append([]string{os.Args[0]}, flag.Args()...) // Emulate os.Args by prepending the cmd to list of args coming from flag
-	cmd, cmdErr := console.GetCommandValue(args)
-	if cmdErr != nil {
-		log.Fatal(cmdErr)
-	}
-	res := cmd.RpcResult()
-	if rpcErr := client.Call(cmd.RpcMethod(), cmd.RpcParams(), res); rpcErr != nil {
-	}
-	fmt.Println("Result:", res)
 
+	if len(flag.Args()) != 0 {
+		executeCommand(strings.Join(flag.Args(), " "))
+		return
+	}
+
+	fmt.Println("Welcome to CGRateS console!")
+	fmt.Print("Type `help` for a list of commands\n\n")
+
+	line := liner.NewLiner()
+	defer line.Close()
+
+	line.SetCompleter(func(line string) (comp []string) {
+		commands := console.GetCommands()
+		for name, cmd := range commands {
+			if strings.HasPrefix(name, strings.ToLower(line)) {
+				comp = append(comp, name)
+			}
+			// try arguments
+			if strings.HasPrefix(line, name) {
+				// get last word
+				lastSpace := strings.LastIndex(line, " ")
+				lastSpace += 1
+				for _, arg := range cmd.ClientArgs() {
+					if strings.HasPrefix(arg, line[lastSpace:]) {
+						comp = append(comp, line[:lastSpace]+arg)
+					}
+				}
+			}
+		}
+		return
+	})
+
+	if f, err := os.Open(history_fn); err == nil {
+		line.ReadHistory(f)
+		f.Close()
+	}
+
+	stop := false
+	for !stop {
+		if command, err := line.Prompt("cgr> "); err != nil {
+			if err == io.EOF {
+				fmt.Println("\nbye!")
+				stop = true
+			} else {
+				fmt.Print("Error reading line: ", err)
+			}
+		} else {
+			line.AppendHistory(command)
+			switch strings.ToLower(strings.TrimSpace(command)) {
+			case "quit", "exit", "bye", "close":
+				fmt.Println("\nbye!")
+				stop = true
+			default:
+				executeCommand(command)
+			}
+		}
+	}
+
+	if f, err := os.Create(history_fn); err != nil {
+		log.Print("Error writing history file: ", err)
+	} else {
+		line.WriteHistory(f)
+		f.Close()
+	}
 }

cmd/cgr-engine/cgr-engine.go

@@ -0,0 +1,669 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"reflect"
+	"runtime"
+	"runtime/pprof"
+	"strconv"
+	"time"
+
+	"github.com/cgrates/cgrates/apier/v1"
+	"github.com/cgrates/cgrates/apier/v2"
+	"github.com/cgrates/cgrates/balancer2go"
+	"github.com/cgrates/cgrates/cdrc"
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/history"
+	"github.com/cgrates/cgrates/scheduler"
+	"github.com/cgrates/cgrates/sessionmanager"
+	"github.com/cgrates/cgrates/utils"
+	"github.com/cgrates/rpcclient"
+)
+
+const (
+	JSON     = "json"
+	GOB      = "gob"
+	POSTGRES = "postgres"
+	MYSQL    = "mysql"
+	MONGO    = "mongo"
+	REDIS    = "redis"
+	SAME     = "same"
+	FS       = "freeswitch"
+	KAMAILIO = "kamailio"
+	OSIPS    = "opensips"
+)
+
+var (
+	cfgDir       = flag.String("config_dir", utils.CONFIG_DIR, "Configuration directory path.")
+	version      = flag.Bool("version", false, "Prints the application version.")
+	raterEnabled = flag.Bool("rater", false, "Enforce starting of the rater daemon overwriting config")
+	schedEnabled = flag.Bool("scheduler", false, "Enforce starting of the scheduler daemon .overwriting config")
+	cdrsEnabled  = flag.Bool("cdrs", false, "Enforce starting of the cdrs daemon overwriting config")
+	pidFile      = flag.String("pid", "", "Write pid file")
+	cpuprofile   = flag.String("cpuprofile", "", "write cpu profile to file")
+	singlecpu    = flag.Bool("singlecpu", false, "Run on single CPU core")
+
+	cfg   *config.CGRConfig
+	sms   []sessionmanager.SessionManager
+	smRpc *v1.SessionManagerV1
+	err   error
+)
+
+func startCdrcs(internalCdrSChan chan *engine.CdrServer, internalRaterChan chan *engine.Responder, exitChan chan bool) {
+	for {
+		cdrcChildrenChan := make(chan struct{}) // Will use it to communicate with the children of one fork
+		for _, cdrcCfgs := range cfg.CdrcProfiles {
+			var cdrcCfg *config.CdrcConfig
+			for _, cdrcCfg = range cdrcCfgs { // Take a random config out since they should be the same
+				break
+			}
+			if cdrcCfg.Enabled == false {
+				continue // Ignore not enabled
+			}
+			go startCdrc(internalCdrSChan, internalRaterChan, cdrcCfgs, cfg.HttpSkipTlsVerify, cdrcChildrenChan, exitChan)
+		}
+		select {
+		case <-exitChan: // Stop forking CDRCs
+			break
+		case <-cfg.ConfigReloads[utils.CDRC]: // Consume the load request and wait for a new one
+			close(cdrcChildrenChan) // Stop all the children of the previous run
+			engine.Logger.Info("<CDRC> Configuration reload")
+			continue
+		}
+	}
+}
+
+// Fires up a cdrc instance
+func startCdrc(internalCdrSChan chan *engine.CdrServer, internalRaterChan chan *engine.Responder, cdrcCfgs map[string]*config.CdrcConfig, httpSkipTlsCheck bool,
+	closeChan chan struct{}, exitChan chan bool) {
+	var cdrsConn engine.Connector
+	var cdrcCfg *config.CdrcConfig
+	for _, cdrcCfg = range cdrcCfgs { // Take the first config out, does not matter which one
+		break
+	}
+	if cdrcCfg.Cdrs == utils.INTERNAL {
+		cdrsChan := <-internalCdrSChan // This will signal that the cdrs part is populated in internalRaterChan
+		internalCdrSChan <- cdrsChan   // Put it back for other components
+		resp := <-internalRaterChan
+		cdrsConn = resp
+		internalRaterChan <- resp
+	} else {
+		conn, err := rpcclient.NewRpcClient("tcp", cdrcCfg.Cdrs, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<CDRC> Could not connect to CDRS via RPC: %v", err))
+			exitChan <- true
+			return
+		}
+		cdrsConn = &engine.RPCClientConnector{Client: conn}
+	}
+	cdrc, err := cdrc.NewCdrc(cdrcCfgs, httpSkipTlsCheck, cdrsConn, closeChan, cfg.DefaultTimezone)
+	if err != nil {
+		engine.Logger.Crit(fmt.Sprintf("Cdrc config parsing error: %s", err.Error()))
+		exitChan <- true
+		return
+	}
+	if err := cdrc.Run(); err != nil {
+		engine.Logger.Crit(fmt.Sprintf("Cdrc run error: %s", err.Error()))
+		exitChan <- true // If run stopped, something is bad, stop the application
+	}
+}
+
+func startSmFreeSWITCH(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) {
+	engine.Logger.Info("Starting CGRateS SM-FreeSWITCH service.")
+	var raterConn, cdrsConn engine.Connector
+	var client *rpcclient.RpcClient
+	var err error
+	// Connect to rater
+	for _, raterCfg := range cfg.SmFsConfig.HaRater {
+		if raterCfg.Server == utils.INTERNAL {
+			resp := <-internalRaterChan
+			raterConn = resp // Will overwrite here for the sake of keeping internally the new configuration format for ha connections
+			internalRaterChan <- resp
+		} else {
+			client, err = rpcclient.NewRpcClient("tcp", raterCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
+			if err != nil { //Connected so no need to reiterate
+				engine.Logger.Crit(fmt.Sprintf("<SM-FreeSWITCH> Could not connect to rater via RPC: %v", err))
+				exitChan <- true
+				return
+			}
+			raterConn = &engine.RPCClientConnector{Client: client}
+		}
+	}
+	// Connect to CDRS
+	if reflect.DeepEqual(cfg.SmFsConfig.HaCdrs, cfg.SmFsConfig.HaRater) {
+		cdrsConn = raterConn
+	} else if len(cfg.SmFsConfig.HaCdrs) != 0 {
+		for _, cdrsCfg := range cfg.SmFsConfig.HaCdrs {
+			if cdrsCfg.Server == utils.INTERNAL {
+				resp := <-internalRaterChan
+				cdrsConn = resp
+				internalRaterChan <- resp
+			} else {
+				client, err = rpcclient.NewRpcClient("tcp", cdrsCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
+				if err != nil {
+					engine.Logger.Crit(fmt.Sprintf("<SM-FreeSWITCH> Could not connect to CDRS via RPC: %v", err))
+					exitChan <- true
+					return
+				}
+				cdrsConn = &engine.RPCClientConnector{Client: client}
+			}
+		}
+	}
+	sm := sessionmanager.NewFSSessionManager(cfg.SmFsConfig, raterConn, cdrsConn, cfg.DefaultTimezone)
+	sms = append(sms, sm)
+	smRpc.SMs = append(smRpc.SMs, sm)
+	if err = sm.Connect(); err != nil {
+		engine.Logger.Err(fmt.Sprintf("<SessionManager> error: %s!", err))
+	}
+	exitChan <- true
+}
+
+func startSmKamailio(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) {
+	engine.Logger.Info("Starting CGRateS SM-Kamailio service.")
+	var raterConn, cdrsConn engine.Connector
+	var client *rpcclient.RpcClient
+	var err error
+	// Connect to rater
+	for _, raterCfg := range cfg.SmKamConfig.HaRater {
+		if raterCfg.Server == utils.INTERNAL {
+			resp := <-internalRaterChan
+			raterConn = resp // Will overwrite here for the sake of keeping internally the new configuration format for ha connections
+			internalRaterChan <- resp
+		} else {
+			client, err = rpcclient.NewRpcClient("tcp", raterCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
+			if err != nil { //Connected so no need to reiterate
+				engine.Logger.Crit(fmt.Sprintf("<SM-FreeSWITCH> Could not connect to rater via RPC: %v", err))
+				exitChan <- true
+				return
+			}
+			raterConn = &engine.RPCClientConnector{Client: client}
+		}
+	}
+	// Connect to CDRS
+	if reflect.DeepEqual(cfg.SmKamConfig.HaCdrs, cfg.SmKamConfig.HaRater) {
+		cdrsConn = raterConn
+	} else if len(cfg.SmKamConfig.HaCdrs) != 0 {
+		for _, cdrsCfg := range cfg.SmKamConfig.HaCdrs {
+			if cdrsCfg.Server == utils.INTERNAL {
+				resp := <-internalRaterChan
+				cdrsConn = resp
+				internalRaterChan <- resp
+			} else {
+				client, err = rpcclient.NewRpcClient("tcp", cdrsCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
+				if err != nil {
+					engine.Logger.Crit(fmt.Sprintf("<SM-FreeSWITCH> Could not connect to CDRS via RPC: %v", err))
+					exitChan <- true
+					return
+				}
+				cdrsConn = &engine.RPCClientConnector{Client: client}
+			}
+		}
+	}
+	sm, _ := sessionmanager.NewKamailioSessionManager(cfg.SmKamConfig, raterConn, cdrsConn, cfg.DefaultTimezone)
+	sms = append(sms, sm)
+	smRpc.SMs = append(smRpc.SMs, sm)
+	if err = sm.Connect(); err != nil {
+		engine.Logger.Err(fmt.Sprintf("<SessionManager> error: %s!", err))
+	}
+	exitChan <- true
+}
+
+func startSmOpenSIPS(internalRaterChan chan *engine.Responder, cdrDb engine.CdrStorage, exitChan chan bool) {
+	engine.Logger.Info("Starting CGRateS SM-OpenSIPS service.")
+	var raterConn, cdrsConn engine.Connector
+	var client *rpcclient.RpcClient
+	var err error
+	// Connect to rater
+	for _, raterCfg := range cfg.SmOsipsConfig.HaRater {
+		if raterCfg.Server == utils.INTERNAL {
+			resp := <-internalRaterChan
+			raterConn = resp // Will overwrite here for the sake of keeping internally the new configuration format for ha connections
+			internalRaterChan <- resp
+		} else {
+			client, err = rpcclient.NewRpcClient("tcp", raterCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
+			if err != nil { //Connected so no need to reiterate
+				engine.Logger.Crit(fmt.Sprintf("<SM-FreeSWITCH> Could not connect to rater via RPC: %v", err))
+				exitChan <- true
+				return
+			}
+			raterConn = &engine.RPCClientConnector{Client: client}
+		}
+	}
+	// Connect to CDRS
+	if reflect.DeepEqual(cfg.SmOsipsConfig.HaCdrs, cfg.SmOsipsConfig.HaRater) {
+		cdrsConn = raterConn
+	} else if len(cfg.SmOsipsConfig.HaCdrs) != 0 {
+		for _, cdrsCfg := range cfg.SmOsipsConfig.HaCdrs {
+			if cdrsCfg.Server == utils.INTERNAL {
+				resp := <-internalRaterChan
+				cdrsConn = resp
+				internalRaterChan <- resp
+			} else {
+				client, err = rpcclient.NewRpcClient("tcp", cdrsCfg.Server, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
+				if err != nil {
+					engine.Logger.Crit(fmt.Sprintf("<SM-FreeSWITCH> Could not connect to CDRS via RPC: %v", err))
+					exitChan <- true
+					return
+				}
+				cdrsConn = &engine.RPCClientConnector{Client: client}
+			}
+		}
+	}
+	sm, _ := sessionmanager.NewOSipsSessionManager(cfg.SmOsipsConfig, cfg.Reconnects, raterConn, cdrsConn, cfg.DefaultTimezone)
+	sms = append(sms, sm)
+	smRpc.SMs = append(smRpc.SMs, sm)
+	if err := sm.Connect(); err != nil {
+		engine.Logger.Err(fmt.Sprintf("<SM-OpenSIPS> error: %s!", err))
+	}
+	exitChan <- true
+}
+
+func startCDRS(internalCdrSChan chan *engine.CdrServer, logDb engine.LogStorage, cdrDb engine.CdrStorage,
+	internalRaterChan chan *engine.Responder, internalPubSubSChan chan engine.PublisherSubscriber,
+	internalUserSChan chan engine.UserService, internalAliaseSChan chan engine.AliasService,
+	internalCdrStatSChan chan engine.StatsInterface, server *engine.Server, exitChan chan bool) {
+	engine.Logger.Info("Starting CGRateS CDRS service.")
+	var err error
+	var client *rpcclient.RpcClient
+	// Rater connection init
+	var raterConn engine.Connector
+	if cfg.CDRSRater == utils.INTERNAL {
+		responder := <-internalRaterChan // Wait for rater to come up before start querying
+		raterConn = responder
+		internalRaterChan <- responder // Put back the connection since there might be other entities waiting for it
+	} else if len(cfg.CDRSRater) != 0 {
+		client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSRater, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
+		if err != nil {
+			engine.Logger.Crit(fmt.Sprintf("<CDRS> Could not connect to rater: %s", err.Error()))
+			exitChan <- true
+			return
+		}
+		raterConn = &engine.RPCClientConnector{Client: client}
+	}
+	// Pubsub connection init
+	var pubSubConn engine.PublisherSubscriber
+	if cfg.CDRSPubSub == utils.INTERNAL {
+		pubSubs := <-internalPubSubSChan
+		pubSubConn = pubSubs
+		internalPubSubSChan <- pubSubs
+	} else if len(cfg.CDRSPubSub) != 0 {
+		if cfg.CDRSRater == cfg.CDRSPubSub {
+			pubSubConn = &engine.ProxyPubSub{Client: client}
+		} else {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSPubSub, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
+			if err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<CDRS> Could not connect to pubsub server: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+			pubSubConn = &engine.ProxyPubSub{Client: client}
+		}
+	}
+	// Users connection init
+	var usersConn engine.UserService
+	if cfg.CDRSUsers == utils.INTERNAL {
+		userS := <-internalUserSChan
+		usersConn = userS
+		internalUserSChan <- userS
+	} else if len(cfg.CDRSUsers) != 0 {
+		if cfg.CDRSRater == cfg.CDRSUsers {
+			usersConn = &engine.ProxyUserService{Client: client}
+		} else {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSUsers, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
+			if err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<CDRS> Could not connect to users server: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+			usersConn = &engine.ProxyUserService{Client: client}
+		}
+	}
+	// Aliases connection init
+	var aliasesConn engine.AliasService
+	if cfg.CDRSAliases == utils.INTERNAL {
+		aliaseS := <-internalAliaseSChan
+		aliasesConn = aliaseS
+		internalAliaseSChan <- aliaseS
+	} else if len(cfg.CDRSAliases) != 0 {
+		if cfg.CDRSRater == cfg.CDRSAliases {
+			aliasesConn = &engine.ProxyAliasService{Client: client}
+		} else {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSAliases, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
+			if err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<CDRS> Could not connect to aliases server: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+			aliasesConn = &engine.ProxyAliasService{Client: client}
+		}
+	}
+	// Stats connection init
+	var statsConn engine.StatsInterface
+	if cfg.CDRSStats == utils.INTERNAL {
+		statS := <-internalCdrStatSChan
+		statsConn = statS
+		internalCdrStatSChan <- statS
+	} else if len(cfg.CDRSStats) != 0 {
+		if cfg.CDRSRater == cfg.CDRSStats {
+			statsConn = &engine.ProxyStats{Client: client}
+		} else {
+			client, err = rpcclient.NewRpcClient("tcp", cfg.CDRSStats, cfg.ConnectAttempts, cfg.Reconnects, utils.GOB)
+			if err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<CDRS> Could not connect to stats server: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+			statsConn = &engine.ProxyStats{Client: client}
+		}
+	}
+
+	cdrServer, _ := engine.NewCdrServer(cfg, cdrDb, raterConn, pubSubConn, usersConn, aliasesConn, statsConn)
+	engine.Logger.Info("Registering CDRS HTTP Handlers.")
+	cdrServer.RegisterHanlersToServer(server)
+	engine.Logger.Info("Registering CDRS RPC service.")
+	cdrSrv := v1.CdrsV1{CdrSrv: cdrServer}
+	server.RpcRegister(&cdrSrv)
+	server.RpcRegister(&v2.CdrsV2{CdrsV1: cdrSrv})
+	// Make the cdr server available for internal communication
+	responder := <-internalRaterChan // Retrieve again the responder
+	responder.CdrSrv = cdrServer     // Attach connection to cdrServer in responder, so it can be used later
+	internalRaterChan <- responder   // Put back the connection for the rest of the system
+	internalCdrSChan <- cdrServer    // Signal that cdrS is operational
+}
+
+func startScheduler(internalSchedulerChan chan *scheduler.Scheduler, ratingDb engine.RatingStorage, exitChan chan bool) {
+	engine.Logger.Info("Starting CGRateS Scheduler.")
+	sched := scheduler.NewScheduler()
+	go reloadSchedulerSingnalHandler(sched, ratingDb)
+	time.Sleep(1)
+	internalSchedulerChan <- sched
+	sched.LoadActionPlans(ratingDb)
+	sched.Loop()
+	exitChan <- true // Should not get out of loop though
+}
+
+func startCdrStats(internalCdrStatSChan chan engine.StatsInterface, ratingDb engine.RatingStorage, accountDb engine.AccountingStorage, server *engine.Server) {
+	cdrStats := engine.NewStats(ratingDb, accountDb, cfg.CDRStatsSaveInterval)
+	server.RpcRegister(cdrStats)
+	server.RpcRegister(&v1.CDRStatsV1{CdrStats: cdrStats}) // Public APIs
+	internalCdrStatSChan <- cdrStats
+}
+
+func startHistoryServer(internalHistorySChan chan history.Scribe, server *engine.Server, exitChan chan bool) {
+	scribeServer, err := history.NewFileScribe(cfg.HistoryDir, cfg.HistorySaveInterval)
+	if err != nil {
+		engine.Logger.Crit(fmt.Sprintf("<HistoryServer> Could not start, error: %s", err.Error()))
+		exitChan <- true
+	}
+	server.RpcRegisterName("ScribeV1", scribeServer)
+	internalHistorySChan <- scribeServer
+}
+
+func startPubSubServer(internalPubSubSChan chan engine.PublisherSubscriber, accountDb engine.AccountingStorage, server *engine.Server) {
+	pubSubServer := engine.NewPubSub(accountDb, cfg.HttpSkipTlsVerify)
+	server.RpcRegisterName("PubSubV1", pubSubServer)
+	internalPubSubSChan <- pubSubServer
+}
+
+// ToDo: Make sure we are caching before starting this one
+func startAliasesServer(internalAliaseSChan chan engine.AliasService, accountDb engine.AccountingStorage, server *engine.Server, exitChan chan bool) {
+	aliasesServer := engine.NewAliasHandler(accountDb)
+	server.RpcRegisterName("AliasesV1", aliasesServer)
+	if err := accountDb.CacheAccountingPrefixes(utils.ALIASES_PREFIX); err != nil {
+		engine.Logger.Crit(fmt.Sprintf("<Aliases> Could not start, error: %s", err.Error()))
+		exitChan <- true
+		return
+	}
+	internalAliaseSChan <- aliasesServer
+}
+
+func startUsersServer(internalUserSChan chan engine.UserService, accountDb engine.AccountingStorage, server *engine.Server, exitChan chan bool) {
+	userServer, err := engine.NewUserMap(accountDb, cfg.UserServerIndexes)
+	if err != nil {
+		engine.Logger.Crit(fmt.Sprintf("<UsersService> Could not start, error: %s", err.Error()))
+		exitChan <- true
+		return
+	}
+	server.RpcRegisterName("UsersV1", userServer)
+	internalUserSChan <- userServer
+}
+
+func startRpc(server *engine.Server, internalRaterChan chan *engine.Responder,
+	internalCdrSChan chan *engine.CdrServer,
+	internalCdrStatSChan chan engine.StatsInterface,
+	internalHistorySChan chan history.Scribe,
+	internalPubSubSChan chan engine.PublisherSubscriber,
+	internalUserSChan chan engine.UserService,
+	internalAliaseSChan chan engine.AliasService) {
+	select { // Any of the rpc methods will unlock listening to rpc requests
+	case resp := <-internalRaterChan:
+		internalRaterChan <- resp
+	case cdrs := <-internalCdrSChan:
+		internalCdrSChan <- cdrs
+	case cdrstats := <-internalCdrStatSChan:
+		internalCdrStatSChan <- cdrstats
+	case hist := <-internalHistorySChan:
+		internalHistorySChan <- hist
+	case pubsubs := <-internalPubSubSChan:
+		internalPubSubSChan <- pubsubs
+	case users := <-internalUserSChan:
+		internalUserSChan <- users
+	case aliases := <-internalAliaseSChan:
+		internalAliaseSChan <- aliases
+	}
+	go server.ServeJSON(cfg.RPCJSONListen)
+	go server.ServeGOB(cfg.RPCGOBListen)
+	go server.ServeHTTP(cfg.HTTPListen)
+}
+
+func writePid() {
+	engine.Logger.Info(*pidFile)
+	f, err := os.Create(*pidFile)
+	if err != nil {
+		log.Fatal("Could not write pid file: ", err)
+	}
+	f.WriteString(strconv.Itoa(os.Getpid()))
+	if err := f.Close(); err != nil {
+		log.Fatal("Could not write pid file: ", err)
+	}
+}
+
+func main() {
+	flag.Parse()
+	if *version {
+		fmt.Println("CGRateS " + utils.VERSION)
+		return
+	}
+	if *pidFile != "" {
+		writePid()
+	}
+	if !*singlecpu {
+		runtime.GOMAXPROCS(runtime.NumCPU()) // For now it slows down computing due to CPU management, to be reviewed in future Go releases
+	}
+	if *cpuprofile != "" {
+		f, err := os.Create(*cpuprofile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+	cfg, err = config.NewCGRConfigFromFolder(*cfgDir)
+	if err != nil {
+		engine.Logger.Crit(fmt.Sprintf("Could not parse config: %s exiting!", err))
+		return
+	}
+	config.SetCgrConfig(cfg) // Share the config object
+	if *raterEnabled {
+		cfg.RaterEnabled = *raterEnabled
+	}
+	if *schedEnabled {
+		cfg.SchedulerEnabled = *schedEnabled
+	}
+	if *cdrsEnabled {
+		cfg.CDRSEnabled = *cdrsEnabled
+	}
+	var ratingDb engine.RatingStorage
+	var accountDb engine.AccountingStorage
+	var logDb engine.LogStorage
+	var loadDb engine.LoadStorage
+	var cdrDb engine.CdrStorage
+	if cfg.RaterEnabled || cfg.SchedulerEnabled { // Only connect to dataDb if necessary
+		ratingDb, err = engine.ConfigureRatingStorage(cfg.TpDbType, cfg.TpDbHost, cfg.TpDbPort,
+			cfg.TpDbName, cfg.TpDbUser, cfg.TpDbPass, cfg.DBDataEncoding)
+		if err != nil { // Cannot configure getter database, show stopper
+			engine.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err))
+			return
+		}
+		defer ratingDb.Close()
+		engine.SetRatingStorage(ratingDb)
+		accountDb, err = engine.ConfigureAccountingStorage(cfg.DataDbType, cfg.DataDbHost, cfg.DataDbPort,
+			cfg.DataDbName, cfg.DataDbUser, cfg.DataDbPass, cfg.DBDataEncoding)
+		if err != nil { // Cannot configure getter database, show stopper
+			engine.Logger.Crit(fmt.Sprintf("Could not configure dataDb: %s exiting!", err))
+			return
+		}
+		defer accountDb.Close()
+		engine.SetAccountingStorage(accountDb)
+	}
+	if cfg.RaterEnabled || cfg.CDRSEnabled || cfg.SchedulerEnabled { // Only connect to storDb if necessary
+		logDb, err = engine.ConfigureLogStorage(cfg.StorDBType, cfg.StorDBHost, cfg.StorDBPort,
+			cfg.StorDBName, cfg.StorDBUser, cfg.StorDBPass, cfg.DBDataEncoding, cfg.StorDBMaxOpenConns, cfg.StorDBMaxIdleConns)
+		if err != nil { // Cannot configure logger database, show stopper
+			engine.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting!", err))
+			return
+		}
+		defer logDb.Close()
+		engine.SetStorageLogger(logDb)
+		// loadDb,cdrDb and logDb are all mapped on the same stordb storage
+		loadDb = logDb.(engine.LoadStorage)
+		cdrDb = logDb.(engine.CdrStorage)
+		engine.SetCdrStorage(cdrDb)
+	}
+
+	engine.SetRoundingDecimals(cfg.RoundingDecimals)
+	stopHandled := false
+
+	// Rpc/http server
+	server := new(engine.Server)
+
+	// Async starts here, will follow cgrates.json start order
+	exitChan := make(chan bool)
+
+	// Define internal connections via channels
+	internalBalancerChan := make(chan *balancer2go.Balancer, 1)
+	internalRaterChan := make(chan *engine.Responder, 1)
+	internalSchedulerChan := make(chan *scheduler.Scheduler, 1)
+	internalCdrSChan := make(chan *engine.CdrServer, 1)
+	internalCdrStatSChan := make(chan engine.StatsInterface, 1)
+	internalHistorySChan := make(chan history.Scribe, 1)
+	internalPubSubSChan := make(chan engine.PublisherSubscriber, 1)
+	internalUserSChan := make(chan engine.UserService, 1)
+	internalAliaseSChan := make(chan engine.AliasService, 1)
+
+	// Start balancer service
+	if cfg.BalancerEnabled {
+		go startBalancer(internalBalancerChan, &stopHandled, exitChan) // Not really needed async here but to cope with uniformity
+	}
+
+	// Start rater service
+	if cfg.RaterEnabled {
+		go startRater(internalRaterChan, internalBalancerChan, internalSchedulerChan, internalCdrStatSChan, internalHistorySChan, internalPubSubSChan, internalUserSChan, internalAliaseSChan,
+			server, ratingDb, accountDb, loadDb, cdrDb, logDb, &stopHandled, exitChan)
+	}
+
+	// Start Scheduler
+	if cfg.SchedulerEnabled {
+		go startScheduler(internalSchedulerChan, ratingDb, exitChan)
+	}
+
+	// Start CDR Server
+	if cfg.CDRSEnabled {
+		go startCDRS(internalCdrSChan, logDb, cdrDb, internalRaterChan, internalPubSubSChan, internalUserSChan, internalAliaseSChan, internalCdrStatSChan, server, exitChan)
+	}
+
+	// Start CDR Stats server
+	if cfg.CDRStatsEnabled {
+		go startCdrStats(internalCdrStatSChan, ratingDb, accountDb, server)
+	}
+
+	// Start CDRC components if necessary
+	go startCdrcs(internalCdrSChan, internalRaterChan, exitChan)
+
+	// Start SM-FreeSWITCH
+	if cfg.SmFsConfig.Enabled {
+		go startSmFreeSWITCH(internalRaterChan, cdrDb, exitChan)
+		// close all sessions on shutdown
+		go shutdownSessionmanagerSingnalHandler(exitChan)
+	}
+
+	// Start SM-Kamailio
+	if cfg.SmKamConfig.Enabled {
+		go startSmKamailio(internalRaterChan, cdrDb, exitChan)
+	}
+
+	// Start SM-OpenSIPS
+	if cfg.SmOsipsConfig.Enabled {
+		go startSmOpenSIPS(internalRaterChan, cdrDb, exitChan)
+	}
+
+	// Register session manager service // FixMe: make sure this is thread safe
+	if cfg.SmFsConfig.Enabled || cfg.SmKamConfig.Enabled || cfg.SmOsipsConfig.Enabled { // Register SessionManagerV1 service
+		smRpc = new(v1.SessionManagerV1)
+		server.RpcRegister(smRpc)
+	}
+
+	// Start HistoryS service
+	if cfg.HistoryServerEnabled {
+		go startHistoryServer(internalHistorySChan, server, exitChan)
+	}
+
+	// Start PubSubS service
+	if cfg.PubSubServerEnabled {
+		go startPubSubServer(internalPubSubSChan, accountDb, server)
+	}
+
+	// Start Aliases service
+	if cfg.AliasesServerEnabled {
+		go startAliasesServer(internalAliaseSChan, accountDb, server, exitChan)
+	}
+
+	// Start users service
+	if cfg.UserServerEnabled {
+		go startUsersServer(internalUserSChan, accountDb, server, exitChan)
+	}
+
+	// Serve rpc connections
+	go startRpc(server, internalRaterChan, internalCdrSChan, internalCdrStatSChan, internalHistorySChan,
+		internalPubSubSChan, internalUserSChan, internalAliaseSChan)
+	<-exitChan
+
+	if *pidFile != "" {
+		if err := os.Remove(*pidFile); err != nil {
+			engine.Logger.Warning("Could not remove pid file: " + err.Error())
+		}
+	}
+	engine.Logger.Info("Stopped all components. CGRateS shutdown!")
+}

cmd/cgr-engine/engine_test.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

cmd/cgr-engine/rater.go

@@ -0,0 +1,250 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/cgrates/cgrates/apier/v1"
+	"github.com/cgrates/cgrates/apier/v2"
+	"github.com/cgrates/cgrates/balancer2go"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/history"
+	"github.com/cgrates/cgrates/scheduler"
+	"github.com/cgrates/cgrates/utils"
+)
+
+func startBalancer(internalBalancerChan chan *balancer2go.Balancer, stopHandled *bool, exitChan chan bool) {
+	bal := balancer2go.NewBalancer()
+	go stopBalancerSignalHandler(bal, exitChan)
+	*stopHandled = true
+	internalBalancerChan <- bal
+}
+
+// Starts rater and reports on chan
+func startRater(internalRaterChan chan *engine.Responder, internalBalancerChan chan *balancer2go.Balancer, internalSchedulerChan chan *scheduler.Scheduler,
+	internalCdrStatSChan chan engine.StatsInterface, internalHistorySChan chan history.Scribe,
+	internalPubSubSChan chan engine.PublisherSubscriber, internalUserSChan chan engine.UserService, internalAliaseSChan chan engine.AliasService,
+	server *engine.Server,
+	ratingDb engine.RatingStorage, accountDb engine.AccountingStorage, loadDb engine.LoadStorage, cdrDb engine.CdrStorage, logDb engine.LogStorage,
+	stopHandled *bool, exitChan chan bool) {
+	waitTasks := make([]chan struct{}, 0)
+
+	//Cache load
+	cacheTaskChan := make(chan struct{})
+	waitTasks = append(waitTasks, cacheTaskChan)
+	go func() {
+		defer close(cacheTaskChan)
+		if err := ratingDb.CacheRatingAll(); err != nil {
+			engine.Logger.Crit(fmt.Sprintf("Cache rating error: %s", err.Error()))
+			exitChan <- true
+			return
+		}
+		if err := accountDb.CacheAccountingPrefixes(); err != nil { // Used to cache load history
+			engine.Logger.Crit(fmt.Sprintf("Cache accounting error: %s", err.Error()))
+			exitChan <- true
+			return
+		}
+
+	}()
+
+	// Retrieve scheduler for it's API methods
+	var sched *scheduler.Scheduler // Need the scheduler in APIer
+	if cfg.SchedulerEnabled {
+		schedTaskChan := make(chan struct{})
+		waitTasks = append(waitTasks, schedTaskChan)
+		go func() {
+			defer close(schedTaskChan)
+			select {
+			case sched = <-internalSchedulerChan:
+				internalSchedulerChan <- sched
+			case <-time.After(cfg.InternalTtl):
+				engine.Logger.Crit("<Rater>: Internal scheduler connection timeout.")
+				exitChan <- true
+				return
+			}
+
+		}()
+	}
+
+	// Connection to balancer
+	var bal *balancer2go.Balancer
+	if cfg.RaterBalancer != "" {
+		balTaskChan := make(chan struct{})
+		waitTasks = append(waitTasks, balTaskChan)
+		go func() {
+			defer close(balTaskChan)
+			if cfg.RaterBalancer == utils.INTERNAL {
+				select {
+				case bal = <-internalBalancerChan:
+					internalBalancerChan <- bal // Put it back if someone else is interested about
+				case <-time.After(cfg.InternalTtl):
+					engine.Logger.Crit("<Rater>: Internal balancer connection timeout.")
+					exitChan <- true
+					return
+				}
+			} else {
+				go registerToBalancer(exitChan)
+				go stopRaterSignalHandler(internalCdrStatSChan, exitChan)
+				*stopHandled = true
+			}
+		}()
+	}
+
+	// Connection to CDRStats
+	var cdrStats engine.StatsInterface
+	if cfg.RaterCdrStats != "" {
+		cdrstatTaskChan := make(chan struct{})
+		waitTasks = append(waitTasks, cdrstatTaskChan)
+		go func() {
+			defer close(cdrstatTaskChan)
+			if cfg.RaterCdrStats == utils.INTERNAL {
+				select {
+				case cdrStats = <-internalCdrStatSChan:
+					internalCdrStatSChan <- cdrStats
+				case <-time.After(cfg.InternalTtl):
+					engine.Logger.Crit("<Rater>: Internal cdrstats connection timeout.")
+					exitChan <- true
+					return
+				}
+			} else if cdrStats, err = engine.NewProxyStats(cfg.RaterCdrStats, cfg.ConnectAttempts, -1); err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<Rater> Could not connect to cdrstats, error: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+		}()
+	}
+
+	// Connection to HistoryS
+	if cfg.RaterHistoryServer != "" {
+		histTaskChan := make(chan struct{})
+		waitTasks = append(waitTasks, histTaskChan)
+		go func() {
+			defer close(histTaskChan)
+			var scribeServer history.Scribe
+			if cfg.RaterHistoryServer == utils.INTERNAL {
+				select {
+				case scribeServer = <-internalHistorySChan:
+					internalHistorySChan <- scribeServer
+				case <-time.After(cfg.InternalTtl):
+					engine.Logger.Crit("<Rater>: Internal historys connection timeout.")
+					exitChan <- true
+					return
+				}
+			} else if scribeServer, err = history.NewProxyScribe(cfg.RaterHistoryServer, cfg.ConnectAttempts, -1); err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<Rater> Could not connect historys, error: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+			engine.SetHistoryScribe(scribeServer) // ToDo: replace package sharing with connection based one
+		}()
+	}
+
+	// Connection to pubsubs
+	if cfg.RaterPubSubServer != "" {
+		pubsubTaskChan := make(chan struct{})
+		waitTasks = append(waitTasks, pubsubTaskChan)
+		go func() {
+			defer close(pubsubTaskChan)
+			var pubSubServer engine.PublisherSubscriber
+			if cfg.RaterPubSubServer == utils.INTERNAL {
+				select {
+				case pubSubServer = <-internalPubSubSChan:
+					internalPubSubSChan <- pubSubServer
+				case <-time.After(cfg.InternalTtl):
+					engine.Logger.Crit("<Rater>: Internal pubsub connection timeout.")
+					exitChan <- true
+					return
+				}
+			} else if pubSubServer, err = engine.NewProxyPubSub(cfg.RaterPubSubServer, cfg.ConnectAttempts, -1); err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<Rater> Could not connect to pubsubs: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+			engine.SetPubSub(pubSubServer) // ToDo: replace package sharing with connection based one
+		}()
+	}
+
+	// Connection to AliasService
+	if cfg.RaterAliasesServer != "" {
+		aliasesTaskChan := make(chan struct{})
+		waitTasks = append(waitTasks, aliasesTaskChan)
+		go func() {
+			defer close(aliasesTaskChan)
+			var aliasesServer engine.AliasService
+			if cfg.RaterAliasesServer == utils.INTERNAL {
+				select {
+				case aliasesServer = <-internalAliaseSChan:
+					internalAliaseSChan <- aliasesServer
+				case <-time.After(cfg.InternalTtl):
+					engine.Logger.Crit("<Rater>: Internal aliases connection timeout.")
+					exitChan <- true
+					return
+				}
+			} else if aliasesServer, err = engine.NewProxyAliasService(cfg.RaterAliasesServer, cfg.ConnectAttempts, -1); err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<Rater> Could not connect to aliases, error: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+			engine.SetAliasService(aliasesServer) // ToDo: replace package sharing with connection based one
+		}()
+	}
+
+	// Connection to UserService
+	var userServer engine.UserService
+	if cfg.RaterUserServer != "" {
+		usersTaskChan := make(chan struct{})
+		waitTasks = append(waitTasks, usersTaskChan)
+		go func() {
+			defer close(usersTaskChan)
+			if cfg.RaterUserServer == utils.INTERNAL {
+				select {
+				case userServer = <-internalUserSChan:
+					internalUserSChan <- userServer
+				case <-time.After(cfg.InternalTtl):
+					engine.Logger.Crit("<Rater>: Internal users connection timeout.")
+					exitChan <- true
+					return
+				}
+			} else if userServer, err = engine.NewProxyUserService(cfg.RaterUserServer, cfg.ConnectAttempts, -1); err != nil {
+				engine.Logger.Crit(fmt.Sprintf("<Rater> Could not connect users, error: %s", err.Error()))
+				exitChan <- true
+				return
+			}
+			engine.SetUserService(userServer)
+		}()
+	}
+
+	// Wait for all connections to complete before going further
+	for _, chn := range waitTasks {
+		<-chn
+	}
+
+	responder := &engine.Responder{Bal: bal, ExitChan: exitChan, Stats: cdrStats}
+	apierRpcV1 := &v1.ApierV1{StorDb: loadDb, RatingDb: ratingDb, AccountDb: accountDb, CdrDb: cdrDb, LogDb: logDb, Sched: sched,
+		Config: cfg, Responder: responder, CdrStatsSrv: cdrStats, Users: userServer}
+	apierRpcV2 := &v2.ApierV2{
+		ApierV1: *apierRpcV1}
+	// internalSchedulerChan shared here
+	server.RpcRegister(responder)
+	server.RpcRegister(apierRpcV1)
+	server.RpcRegister(apierRpcV2)
+	internalRaterChan <- responder // Rater done
+}

cmd/cgr-engine/registration.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -20,88 +20,111 @@ package main
 
 import (
 	"fmt"
-	"github.com/cgrates/cgrates/rater"
-	"github.com/cgrates/cgrates/scheduler"
 	"net/rpc"
 	"os"
 	"os/signal"
 	"syscall"
+
+	"github.com/cgrates/cgrates/balancer2go"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/scheduler"
 )
 
 /*
-Listens for SIGTERM, SIGINT, SIGQUIT system signals and shuts down all the registered raters.
+Listens for SIGTERM, SIGINT, SIGQUIT system signals and shuts down all the registered engines.
 */
-func stopBalancerSingnalHandler() {
+func stopBalancerSignalHandler(bal *balancer2go.Balancer, exitChan chan bool) {
+	c := make(chan os.Signal)
+	signal.Notify(c, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT)
+	sig := <-c
+	engine.Logger.Info(fmt.Sprintf("Caught signal %v, sending shutdown to engines\n", sig))
+	bal.Shutdown("Responder.Shutdown")
+	exitChan <- true
+}
+
+func generalSignalHandler(internalCdrStatSChan chan engine.StatsInterface, exitChan chan bool) {
 	c := make(chan os.Signal)
 	signal.Notify(c, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT)
 
 	sig := <-c
-	rater.Logger.Info(fmt.Sprintf("Caught signal %v, sending shutdownto raters\n", sig))
-	bal.Shutdown("Responder.Shutdown")
+	engine.Logger.Info(fmt.Sprintf("Caught signal %v, shuting down cgr-engine\n", sig))
+	var dummyInt int
+	select {
+	case cdrStats := <-internalCdrStatSChan:
+		cdrStats.Stop(dummyInt, &dummyInt)
+	default:
+	}
+
 	exitChan <- true
 }
 
 /*
 Listens for the SIGTERM, SIGINT, SIGQUIT system signals and  gracefuly unregister from balancer and closes the storage before exiting.
 */
-func stopRaterSingnalHandler() {
+func stopRaterSignalHandler(internalCdrStatSChan chan engine.StatsInterface, exitChan chan bool) {
 	c := make(chan os.Signal)
 	signal.Notify(c, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT)
 	sig := <-c
 
-	rater.Logger.Info(fmt.Sprintf("Caught signal %v, unregistering from balancer\n", sig))
-	unregisterFromBalancer()
+	engine.Logger.Info(fmt.Sprintf("Caught signal %v, unregistering from balancer\n", sig))
+	unregisterFromBalancer(exitChan)
+	var dummyInt int
+	select {
+	case cdrStats := <-internalCdrStatSChan:
+		cdrStats.Stop(dummyInt, &dummyInt)
+	default:
+	}
 	exitChan <- true
 }
 
 /*
 Connects to the balancer and calls unregister RPC method.
 */
-func unregisterFromBalancer() {
+func unregisterFromBalancer(exitChan chan bool) {
 	client, err := rpc.Dial("tcp", cfg.RaterBalancer)
 	if err != nil {
-		rater.Logger.Crit("Cannot contact the balancer!")
+		engine.Logger.Crit("Cannot contact the balancer!")
 		exitChan <- true
 		return
 	}
 	var reply int
-	rater.Logger.Info(fmt.Sprintf("Unregistering from balancer %s", cfg.RaterBalancer))
-	client.Call("Responder.UnRegisterRater", cfg.RaterListen, &reply)
+	engine.Logger.Info(fmt.Sprintf("Unregistering from balancer %s", cfg.RaterBalancer))
+	client.Call("Responder.UnRegisterRater", cfg.RPCGOBListen, &reply)
 	if err := client.Close(); err != nil {
-		rater.Logger.Crit("Could not close balancer unregistration!")
+		engine.Logger.Crit("Could not close balancer unregistration!")
 		exitChan <- true
 	}
 }
 
 /*
-Connects to the balancer and rehisters the rater to the server.
+Connects to the balancer and rehisters the engine to the server.
 */
-func registerToBalancer() {
+func registerToBalancer(exitChan chan bool) {
 	client, err := rpc.Dial("tcp", cfg.RaterBalancer)
 	if err != nil {
-		rater.Logger.Crit(fmt.Sprintf("Cannot contact the balancer: %v", err))
+		engine.Logger.Crit(fmt.Sprintf("Cannot contact the balancer: %v", err))
 		exitChan <- true
 		return
 	}
 	var reply int
-	rater.Logger.Info(fmt.Sprintf("Registering to balancer %s", cfg.RaterBalancer))
-	client.Call("Responder.RegisterRater", cfg.RaterListen, &reply)
+	engine.Logger.Info(fmt.Sprintf("Registering to balancer %s", cfg.RaterBalancer))
+	client.Call("Responder.RegisterRater", cfg.RPCGOBListen, &reply)
 	if err := client.Close(); err != nil {
-		rater.Logger.Crit("Could not close balancer registration!")
+		engine.Logger.Crit("Could not close balancer registration!")
 		exitChan <- true
 	}
-	rater.Logger.Info("Registration finished!")
+	engine.Logger.Info("Registration finished!")
 }
 
 // Listens for the HUP system signal and gracefuly reloads the timers from database.
-func reloadSchedulerSingnalHandler(sched *scheduler.Scheduler, getter rater.DataStorage) {
+func reloadSchedulerSingnalHandler(sched *scheduler.Scheduler, getter engine.RatingStorage) {
 	for {
 		c := make(chan os.Signal)
 		signal.Notify(c, syscall.SIGHUP)
 		sig := <-c
 
-		rater.Logger.Info(fmt.Sprintf("Caught signal %v, reloading action timings.\n", sig))
-		sched.LoadActionTimings(getter)
+		engine.Logger.Info(fmt.Sprintf("Caught signal %v, reloading action timings.\n", sig))
+		sched.LoadActionPlans(getter)
 		// check the tip of the queue for new actions
 		sched.Restart()
 	}
@@ -110,13 +133,15 @@ func reloadSchedulerSingnalHandler(sched *scheduler.Scheduler, getter rater.Data
 /*
 Listens for the SIGTERM, SIGINT, SIGQUIT system signals and shuts down the session manager.
 */
-func shutdownSessionmanagerSingnalHandler() {
+func shutdownSessionmanagerSingnalHandler(exitChan chan bool) {
 	c := make(chan os.Signal)
 	signal.Notify(c, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT)
 	<-c
 
-	if err := sm.Shutdown(); err != nil {
-		rater.Logger.Warning(fmt.Sprintf("<SessionManager> %s", err))
+	for _, sm := range sms {
+		if err := sm.Shutdown(); err != nil {
+			engine.Logger.Warning(fmt.Sprintf("<SessionManager> %s", err))
+		}
 	}
 	exitChan <- true
 }

cmd/cgr-loader/cgr-loader.go

@@ -1,6 +1,6 @@
 /*
 Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Copyright (C) 2012-2015 ITsysCOM
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -21,155 +21,285 @@ package main
 import (
 	"flag"
 	"fmt"
-	"github.com/cgrates/cgrates/rater"
 	"log"
+	"net/rpc"
 	"path"
-	"regexp"
-	"strconv"
-)
 
-const (
-	POSTGRES = "postgres"
-	MONGO    = "mongo"
-	REDIS    = "redis"
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/history"
+	"github.com/cgrates/cgrates/utils"
 )
 
 var (
 	//separator = flag.String("separator", ",", "Default field separator")
-	db_type = flag.String("dbtype", REDIS, "The type of the database (redis|mongo|postgres)")
-	db_host = flag.String("dbhost", "localhost", "The database host to connect to.")
-	db_port = flag.String("dbport", "6379", "The database port to bind to.")
-	db_name = flag.String("dbname", "10", "he name/number of the database to connect to.")
-	db_user = flag.String("dbuser", "", "The database user to sign in as.")
-	db_pass = flag.String("dbpass", "", "The database user's password.")
+	cgrConfig, _ = config.NewDefaultCGRConfig()
+	tpdb_type    = flag.String("tpdb_type", cgrConfig.TpDbType, "The type of the TariffPlan database <redis>")
+	tpdb_host    = flag.String("tpdb_host", cgrConfig.TpDbHost, "The TariffPlan host to connect to.")
+	tpdb_port    = flag.String("tpdb_port", cgrConfig.TpDbPort, "The TariffPlan port to bind to.")
+	tpdb_name    = flag.String("tpdb_name", cgrConfig.TpDbName, "The name/number of the TariffPlan to connect to.")
+	tpdb_user    = flag.String("tpdb_user", cgrConfig.TpDbUser, "The TariffPlan user to sign in as.")
+	tpdb_pass    = flag.String("tpdb_passwd", cgrConfig.TpDbPass, "The TariffPlan user's password.")
 
-	flush    = flag.Bool("flush", false, "Flush the database before importing")
-	dataPath = flag.String("path", ".", "The path containing the data files")
-	version  = flag.Bool("version", false, "Prints the application version.")
+	datadb_type = flag.String("datadb_type", cgrConfig.DataDbType, "The type of the DataDb database <redis>")
+	datadb_host = flag.String("datadb_host", cgrConfig.DataDbHost, "The DataDb host to connect to.")
+	datadb_port = flag.String("datadb_port", cgrConfig.DataDbPort, "The DataDb port to bind to.")
+	datadb_name = flag.String("datadb_name", cgrConfig.DataDbName, "The name/number of the DataDb to connect to.")
+	datadb_user = flag.String("datadb_user", cgrConfig.DataDbUser, "The DataDb user to sign in as.")
+	datadb_pass = flag.String("datadb_passwd", cgrConfig.DataDbPass, "The DataDb user's password.")
 
-	destinationsFn   = "Destinations.csv"
-	ratesFn          = "Rates.csv"
-	timingsFn        = "Timings.csv"
-	ratetimingsFn    = "RateTimings.csv"
-	ratingprofilesFn = "RatingProfiles.csv"
-	actionsFn        = "Actions.csv"
-	actiontimingsFn  = "ActionTimings.csv"
-	actiontriggersFn = "ActionTriggers.csv"
-	accountactionsFn = "AccountActions.csv"
-	sep              rune
+	stor_db_type = flag.String("stordb_type", cgrConfig.StorDBType, "The type of the storDb database <mysql>")
+	stor_db_host = flag.String("stordb_host", cgrConfig.StorDBHost, "The storDb host to connect to.")
+	stor_db_port = flag.String("stordb_port", cgrConfig.StorDBPort, "The storDb port to bind to.")
+	stor_db_name = flag.String("stordb_name", cgrConfig.StorDBName, "The name/number of the storDb to connect to.")
+	stor_db_user = flag.String("stordb_user", cgrConfig.StorDBUser, "The storDb user to sign in as.")
+	stor_db_pass = flag.String("stordb_passwd", cgrConfig.StorDBPass, "The storDb user's password.")
+
+	dbdata_encoding = flag.String("dbdata_encoding", cgrConfig.DBDataEncoding, "The encoding used to store object data in strings")
+
+	flush           = flag.Bool("flushdb", false, "Flush the database before importing")
+	tpid            = flag.String("tpid", "", "The tariff plan id from the database")
+	dataPath        = flag.String("path", "./", "The path to folder containing the data files")
+	version         = flag.Bool("version", false, "Prints the application version.")
+	verbose         = flag.Bool("verbose", false, "Enable detailed verbose logging output")
+	dryRun          = flag.Bool("dry_run", false, "When true will not save loaded data to dataDb but just parse it for consistency and errors.")
+	validate        = flag.Bool("validate", false, "When true will run various check on the loaded data to check for structural errors")
+	stats           = flag.Bool("stats", false, "Generates statsistics about given data.")
+	fromStorDb      = flag.Bool("from_stordb", false, "Load the tariff plan from storDb to dataDb")
+	toStorDb        = flag.Bool("to_stordb", false, "Import the tariff plan from files to storDb")
+	historyServer   = flag.String("history_server", cgrConfig.RPCGOBListen, "The history server address:port, empty to disable automaticautomatic  history archiving")
+	raterAddress    = flag.String("rater_address", cgrConfig.RPCGOBListen, "Rater service to contact for cache reloads, empty to disable automatic cache reloads")
+	cdrstatsAddress = flag.String("cdrstats_address", cgrConfig.RPCGOBListen, "CDRStats service to contact for data reloads, empty to disable automatic data reloads")
+	usersAddress    = flag.String("users_address", cgrConfig.RPCGOBListen, "Users service to contact for data reloads, empty to disable automatic data reloads")
+	runId           = flag.String("runid", "", "Uniquely identify an import/load, postpended to some automatic fields")
+	loadHistorySize = flag.Int("load_history_size", cgrConfig.LoadHistorySize, "Limit the number of records in the load history")
+	timezone        = flag.String("timezone", cgrConfig.DefaultTimezone, `Timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB>`)
 )
 
-type validator struct {
-	fn      string
-	re      *regexp.Regexp
-	message string
-}
-
 func main() {
 	flag.Parse()
 	if *version {
-		fmt.Println("CGRateS " + rater.VERSION)
+		fmt.Println("CGRateS " + utils.VERSION)
 		return
 	}
-	dataFilesValidators := []*validator{
-		&validator{destinationsFn,
-			regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+.?\d*){1}$`),
-			"Tag[0-9A-Za-z_],Prefix[0-9]"},
-		&validator{ratesFn,
-			regexp.MustCompile(`(?:\w+\s*,\s*){2}(?:\d+.?\d*,?){4}$`),
-			"Tag[0-9A-Za-z_],DestinationsTag[0-9A-Za-z_],ConnectFee[0-9.],Price[0-9.],PricedUnits[0-9.],RateIncrement[0-9.]"},
-		&validator{timingsFn,
-			regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\*all\s*,\s*|(?:\d{1,4};?)+\s*,\s*|\s*,\s*){4}(?:\d{2}:\d{2}:\d{2}|\*asap){1}$`),
-			"Tag[0-9A-Za-z_],Years[0-9;]|*all|<empty>,Months[0-9;]|*all|<empty>,MonthDays[0-9;]|*all|<empty>,WeekDays[0-9;]|*all|<empty>,Time[0-9:]|*asap(00:00:00)"},
-		&validator{ratetimingsFn,
-			regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+.?\d*){1}$`),
-			"Tag[0-9A-Za-z_],RatesTag[0-9A-Za-z_],TimingProfile[0-9A-Za-z_],Weight[0-9.]"},
-		&validator{ratingprofilesFn,
-			regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:\d+\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\*all\s*,\s*|[\w:\.]+\s*,\s*){1}(?:\w*\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z){1}$`),
-			"Tenant[0-9A-Za-z_],TOR[0-9],Direction OUT|IN,Subject[0-9A-Za-z_:.]|*all,RatesFallbackSubject[0-9A-Za-z_]|<empty>,RatesTimingTag[0-9A-Za-z_],ActivationTime[[0-9T:X]] (2012-01-01T00:00:00Z)"},
-		&validator{actionsFn,
-			regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\d+\s*,\s*){1}(?:\w+\s*,\s*|\*all\s*,\s*){1}(?:ABSOLUTE\s*,\s*|PERCENT\s*,\s*|\s*,\s*){1}(?:\d*\.?\d*\s*,?\s*){3}$`),
-			"Tag[0-9A-Za-z_],Action[0-9A-Za-z_],BalanceTag[0-9A-Za-z_],Direction OUT|IN,Units[0-9],DestinationTag[0-9A-Za-z_]|*all,PriceType ABSOLUT|PERCENT,PriceValue[0-9.],MinutesWeight[0-9.],Weight[0-9.]"},
-		&validator{actiontimingsFn,
-			regexp.MustCompile(`(?:\w+\s*,\s*){3}(?:\d+\.?\d*){1}`),
-			"Tag[0-9A-Za-z_],ActionsTag[0-9A-Za-z_],TimingTag[0-9A-Za-z_],Weight[0-9.]"},
-		&validator{actiontriggersFn,
-			regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:MONETARY\s*,\s*|SMS\s*,\s*|MINUTES\s*,\s*|INTERNET\s*,\s*|INTERNET_TIME\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\d+\.?\d*\s*,\s*){1}(?:\w+\s*,\s*|\*all\s*,\s*){1}(?:\w+\s*,\s*){1}(?:\d+\.?\d*){1}$`),
-			"Tag[0-9A-Za-z_],BalanceTag MONETARY|SMS|MINUTES|INTERNET|INTERNET_TIME,Direction OUT|IN,ThresholdValue[0-9.],DestinationTag[0-9A-Za-z_]|*all,ActionsTag[0-9A-Za-z_],Weight[0-9.]"},
-		&validator{accountactionsFn,
-			regexp.MustCompile(`(?:\w+\s*,\s*){1}(?:[\w:.]+\s*,\s*){1}(?:OUT\s*,\s*|IN\s*,\s*){1}(?:\w+\s*,?\s*){2}$`),
-			"Tenant[0-9A-Za-z_],Account[0-9A-Za-z_:.],Direction OUT|IN,ActionTimingsTag[0-9A-Za-z_],ActionTriggersTag[0-9A-Za-z_]"},
+	var errRatingDb, errAccDb, errStorDb, err error
+	var ratingDb engine.RatingStorage
+	var accountDb engine.AccountingStorage
+	var storDb engine.LoadStorage
+	var rater, cdrstats, users *rpc.Client
+	var loader engine.LoadReader
+	// Init necessary db connections, only if not already
+	if !*dryRun { // make sure we do not need db connections on dry run, also not importing into any stordb
+		if *fromStorDb {
+			ratingDb, errRatingDb = engine.ConfigureRatingStorage(*tpdb_type, *tpdb_host, *tpdb_port, *tpdb_name,
+				*tpdb_user, *tpdb_pass, *dbdata_encoding)
+			accountDb, errAccDb = engine.ConfigureAccountingStorage(*datadb_type, *datadb_host, *datadb_port, *datadb_name, *datadb_user, *datadb_pass, *dbdata_encoding)
+			storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding,
+				cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns)
+		} else if *toStorDb { // Import from csv files to storDb
+			storDb, errStorDb = engine.ConfigureLoadStorage(*stor_db_type, *stor_db_host, *stor_db_port, *stor_db_name, *stor_db_user, *stor_db_pass, *dbdata_encoding,
+				cgrConfig.StorDBMaxOpenConns, cgrConfig.StorDBMaxIdleConns)
+		} else { // Default load from csv files to dataDb
+			ratingDb, errRatingDb = engine.ConfigureRatingStorage(*tpdb_type, *tpdb_host, *tpdb_port, *tpdb_name,
+				*tpdb_user, *tpdb_pass, *dbdata_encoding)
+			accountDb, errAccDb = engine.ConfigureAccountingStorage(*datadb_type, *datadb_host, *datadb_port, *datadb_name, *datadb_user, *datadb_pass, *dbdata_encoding)
+		}
+		// Defer databases opened to be closed when we are done
+		for _, db := range []engine.Storage{ratingDb, accountDb, storDb} {
+			if db != nil {
+				defer db.Close()
+			}
+		}
+		// Stop on db errors
+		for _, err = range []error{errRatingDb, errAccDb, errStorDb} {
+			if err != nil {
+				log.Fatalf("Could not open database connection: %v", err)
+			}
+		}
+		if *toStorDb { // Import files from a directory into storDb
+			if *tpid == "" {
+				log.Fatal("TPid required, please define it via *-tpid* command argument.")
+			}
+			csvImporter := engine.TPCSVImporter{
+				TPid:     *tpid,
+				StorDb:   storDb,
+				DirPath:  *dataPath,
+				Sep:      ',',
+				Verbose:  *verbose,
+				ImportId: *runId,
+			}
+			if errImport := csvImporter.Run(); errImport != nil {
+				log.Fatal(errImport)
+			}
+			return
+		}
 	}
-	for _, v := range dataFilesValidators {
-		err := rater.ValidateCSVData(path.Join(*dataPath, v.fn), v.re)
+	if *fromStorDb { // Load Tariff Plan from storDb into dataDb
+		loader = storDb
+	} else { // Default load from csv files to dataDb
+		/*for fn, v := range engine.FileValidators {
+			err := engine.ValidateCSVData(path.Join(*dataPath, fn), v.Rule)
+			if err != nil {
+				log.Fatal(err, "\n\t", v.Message)
+			}
+		}*/
+		loader = engine.NewFileCSVStorage(',',
+			path.Join(*dataPath, utils.DESTINATIONS_CSV),
+			path.Join(*dataPath, utils.TIMINGS_CSV),
+			path.Join(*dataPath, utils.RATES_CSV),
+			path.Join(*dataPath, utils.DESTINATION_RATES_CSV),
+			path.Join(*dataPath, utils.RATING_PLANS_CSV),
+			path.Join(*dataPath, utils.RATING_PROFILES_CSV),
+			path.Join(*dataPath, utils.SHARED_GROUPS_CSV),
+			path.Join(*dataPath, utils.LCRS_CSV),
+			path.Join(*dataPath, utils.ACTIONS_CSV),
+			path.Join(*dataPath, utils.ACTION_PLANS_CSV),
+			path.Join(*dataPath, utils.ACTION_TRIGGERS_CSV),
+			path.Join(*dataPath, utils.ACCOUNT_ACTIONS_CSV),
+			path.Join(*dataPath, utils.DERIVED_CHARGERS_CSV),
+			path.Join(*dataPath, utils.CDR_STATS_CSV),
+			path.Join(*dataPath, utils.USERS_CSV),
+			path.Join(*dataPath, utils.ALIASES_CSV),
+		)
+	}
+	tpReader := engine.NewTpReader(ratingDb, accountDb, loader, *tpid, *timezone, *loadHistorySize)
+	err = tpReader.LoadAll()
+	if err != nil {
+		log.Fatal(err)
+	}
+	if *stats {
+		tpReader.ShowStatistics()
+	}
+	if *validate {
+		if !tpReader.IsValid() {
+			return
+		}
+	}
+	if *dryRun { // We were just asked to parse the data, not saving it
+		return
+	}
+	if *historyServer != "" { // Init scribeAgent so we can store the differences
+		if scribeAgent, err := history.NewProxyScribe(*historyServer, 3, 3); err != nil {
+			log.Fatalf("Could not connect to history server, error: %s. Make sure you have properly configured it via -history_server flag.", err.Error())
+			return
+		} else {
+			engine.SetHistoryScribe(scribeAgent)
+			//defer scribeAgent.Client.Close()
+		}
+	} else {
+		log.Print("WARNING: Rates history archiving is disabled!")
+	}
+	if *raterAddress != "" { // Init connection to rater so we can reload it's data
+		rater, err = rpc.Dial("tcp", *raterAddress)
 		if err != nil {
-			log.Fatal(err, "\n\t", v.message)
+			log.Fatalf("Could not connect to rater: %s", err.Error())
+			return
 		}
+	} else {
+		log.Print("WARNING: Rates automatic cache reloading is disabled!")
 	}
-	//sep = []rune(*separator)[0]
-	sep = ','
-	csvr := rater.NewFileCSVReader()
-	err := csvr.LoadDestinations(path.Join(*dataPath, destinationsFn), sep)
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = csvr.LoadRates(path.Join(*dataPath, ratesFn), sep)
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = csvr.LoadTimings(path.Join(*dataPath, timingsFn), sep)
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = csvr.LoadRateTimings(path.Join(*dataPath, ratetimingsFn), sep)
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = csvr.LoadRatingProfiles(path.Join(*dataPath, ratingprofilesFn), sep)
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = csvr.LoadActions(path.Join(*dataPath, actionsFn), sep)
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = csvr.LoadActionTimings(path.Join(*dataPath, actiontimingsFn), sep)
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = csvr.LoadActionTriggers(path.Join(*dataPath, actiontriggersFn), sep)
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = csvr.LoadAccountActions(path.Join(*dataPath, accountactionsFn), sep)
-	if err != nil {
-		log.Fatal(err)
-	}
-	var getter rater.DataStorage
-	switch *db_type {
-	case REDIS:
-		db_nb, err := strconv.Atoi(*db_name)		
-		if err != nil {
-			log.Fatal("Redis db name must be an integer!")
+	if *cdrstatsAddress != "" { // Init connection to rater so we can reload it's data
+		if *cdrstatsAddress == *raterAddress {
+			cdrstats = rater
+		} else {
+			cdrstats, err = rpc.Dial("tcp", *cdrstatsAddress)
+			if err != nil {
+				log.Fatalf("Could not connect to CDRStats API: %s", err.Error())
+				return
+			}
 		}
-		if *db_port != "" {
-			*db_host += ":" + *db_port
-		}
-		getter, err = rater.NewRedisStorage(*db_host, db_nb, *db_pass)
-	case MONGO:
-		getter, err = rater.NewMongoStorage(*db_host, *db_port, *db_name, *db_user, *db_pass)
-	case POSTGRES:
-		getter, err = rater.NewPostgresStorage(*db_host, *db_port, *db_name, *db_user, *db_pass)
-	default:
-		log.Fatal("Unknown data db type, exiting!")
+	} else {
+		log.Print("WARNING: CDRStats automatic data reload is disabled!")
 	}
-
-	if err != nil {
-		log.Fatalf("Could not open database connection: %v", err)
+	if *usersAddress != "" { // Init connection to rater so we can reload it's data
+		if *usersAddress == *raterAddress {
+			users = rater
+		} else {
+			users, err = rpc.Dial("tcp", *usersAddress)
+			if err != nil {
+				log.Fatalf("Could not connect to Users API: %s", err.Error())
+				return
+			}
+		}
+	} else {
+		log.Print("WARNING: Users automatic data reload is disabled!")
 	}
 
 	// write maps to database
-	if err := csvr.WriteToDatabase(getter, *flush, true); err != nil {
+	if err := tpReader.WriteToDatabase(*flush, *verbose); err != nil {
 		log.Fatal("Could not write to database: ", err)
 	}
+	if len(*historyServer) != 0 && *verbose {
+		log.Print("Wrote history.")
+	}
+	// Reload scheduler and cache
+	if rater != nil {
+		reply := ""
+		dstIds, _ := tpReader.GetLoadedIds(utils.DESTINATION_PREFIX)
+		rplIds, _ := tpReader.GetLoadedIds(utils.RATING_PLAN_PREFIX)
+		rpfIds, _ := tpReader.GetLoadedIds(utils.RATING_PROFILE_PREFIX)
+		actIds, _ := tpReader.GetLoadedIds(utils.ACTION_PREFIX)
+		shgIds, _ := tpReader.GetLoadedIds(utils.SHARED_GROUP_PREFIX)
+		aliases, _ := tpReader.GetLoadedIds(utils.ALIASES_PREFIX)
+		lcrIds, _ := tpReader.GetLoadedIds(utils.LCR_PREFIX)
+		dcs, _ := tpReader.GetLoadedIds(utils.DERIVEDCHARGERS_PREFIX)
+		// Reload cache first since actions could be calling info from within
+		if *verbose {
+			log.Print("Reloading cache")
+		}
+		if *flush {
+			dstIds, rplIds, rpfIds, lcrIds = nil, nil, nil, nil // Should reload all these on flush
+		}
+		if err = rater.Call("ApierV1.ReloadCache", utils.ApiReloadCache{
+			DestinationIds:   dstIds,
+			RatingPlanIds:    rplIds,
+			RatingProfileIds: rpfIds,
+			ActionIds:        actIds,
+			SharedGroupIds:   shgIds,
+			Aliases:          aliases,
+			LCRIds:           lcrIds,
+			DerivedChargers:  dcs,
+		}, &reply); err != nil {
+			log.Printf("WARNING: Got error on cache reload: %s\n", err.Error())
+		}
+		actTmgIds, _ := tpReader.GetLoadedIds(utils.ACTION_TIMING_PREFIX)
+		if len(actTmgIds) != 0 {
+			if *verbose {
+				log.Print("Reloading scheduler")
+			}
+			if err = rater.Call("ApierV1.ReloadScheduler", "", &reply); err != nil {
+				log.Printf("WARNING: Got error on scheduler reload: %s\n", err.Error())
+			}
+		}
+
+	}
+	if cdrstats != nil {
+		statsQueueIds, _ := tpReader.GetLoadedIds(utils.CDR_STATS_PREFIX)
+		if *flush {
+			statsQueueIds = []string{} // Force reload all
+		}
+		if len(statsQueueIds) != 0 {
+			if *verbose {
+				log.Print("Reloading CDRStats data")
+			}
+			var reply string
+			if err := cdrstats.Call("CDRStatsV1.ReloadQueues", utils.AttrCDRStatsReloadQueues{StatsQueueIds: statsQueueIds}, &reply); err != nil {
+				log.Printf("WARNING: Failed reloading stat queues, error: %s\n", err.Error())
+			}
+		}
+	}
+
+	if users != nil {
+		userIds, _ := tpReader.GetLoadedIds(utils.USERS_PREFIX)
+		if len(userIds) > 0 {
+			if *verbose {
+				log.Print("Reloading Users data")
+			}
+			var reply string
+			if err := cdrstats.Call("UsersV1.ReloadUsers", "", &reply); err != nil {
+				log.Printf("WARNING: Failed reloading users data, error: %s\n", err.Error())
+			}
+
+		}
+	}
 }

cmd/cgr-rater/cgr-rater.go

@@ -1,328 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package main
-
-import (
-	"errors"
-	"flag"
-	"fmt"
-	"github.com/cgrates/cgrates/balancer2go"
-	"github.com/cgrates/cgrates/config"
-	"github.com/cgrates/cgrates/mediator"
-	"github.com/cgrates/cgrates/rater"
-	"github.com/cgrates/cgrates/scheduler"
-	"github.com/cgrates/cgrates/sessionmanager"
-	"io"
-	"net"
-	"net/rpc"
-	"net/rpc/jsonrpc"
-	"os"
-	"runtime"
-	"strconv"
-	"strings"
-	"time"
-)
-
-const (
-	DISABLED = "disabled"
-	INTERNAL = "internal"
-	JSON     = "json"
-	GOB      = "gob"
-	POSTGRES = "postgres"
-	MONGO    = "mongo"
-	REDIS    = "redis"
-	SAME     = "same"
-	FS       = "freeswitch"
-)
-
-var (
-	cfgPath  = flag.String("config", "/etc/cgrates/cgrates.cfg", "Configuration file location.")
-	version  = flag.Bool("version", false, "Prints the application version.")
-	bal      = balancer2go.NewBalancer()
-	exitChan = make(chan bool)
-	sm       sessionmanager.SessionManager
-	cfg      *config.CGRConfig
-	err      error
-)
-
-func listenToRPCRequests(rpcResponder interface{}, rpcAddress string, rpc_encoding string) {
-	l, err := net.Listen("tcp", rpcAddress)
-	if err != nil {
-		rater.Logger.Crit(fmt.Sprintf("<Rater> Could not listen to %v: %v", rpcAddress, err))
-		exitChan <- true
-		return
-	}
-	defer l.Close()
-
-	rater.Logger.Info(fmt.Sprintf("<Rater> Listening for incomming RPC requests on %v", l.Addr()))
-	rpc.Register(rpcResponder)
-	var serveFunc func(io.ReadWriteCloser)
-	if rpc_encoding == JSON {
-		serveFunc = jsonrpc.ServeConn
-	} else {
-		serveFunc = rpc.ServeConn
-	}
-	for {
-		conn, err := l.Accept()
-		if err != nil {
-			rater.Logger.Err(fmt.Sprintf("<Rater> Accept error: %v", conn))
-			continue
-		}
-
-		rater.Logger.Info(fmt.Sprintf("<Rater> New incoming connection: %v", conn.RemoteAddr()))
-		go serveFunc(conn)
-	}
-}
-
-func startMediator(responder *rater.Responder, loggerDb rater.DataStorage) {
-	var connector rater.Connector
-	if cfg.MediatorRater == INTERNAL {
-		connector = responder
-	} else {
-		var client *rpc.Client
-		var err error
-		if cfg.MediatorRPCEncoding == JSON {
-			client, err = jsonrpc.Dial("tcp", cfg.MediatorRater)
-		} else {
-			client, err = rpc.Dial("tcp", cfg.MediatorRater)
-		}
-		if err != nil {
-			rater.Logger.Crit(fmt.Sprintf("Could not connect to rater: %v", err))
-			exitChan <- true
-		}
-		connector = &rater.RPCClientConnector{Client: client}
-	}
-	if _, err := os.Stat(cfg.MediatorCDRInDir); err != nil {
-		rater.Logger.Crit(fmt.Sprintf("The input path for mediator does not exist: %v", cfg.MediatorCDRInDir))
-		exitChan <- true
-	}
-	if _, err := os.Stat(cfg.MediatorCDROutDir); err != nil {
-		rater.Logger.Crit(fmt.Sprintf("The output path for mediator does not exist: %v", cfg.MediatorCDROutDir))
-		exitChan <- true
-	}
-	// ToDo: Why is here 
-	// Check parsing errors
-	//if cfgParseErr != nil {
-	//	rater.Logger.Crit(fmt.Sprintf("Errors on config parsing: <%v>", cfgParseErr))
-	//	exitChan <- true
-	//}
-
-	m, err := mediator.NewMediator(connector, loggerDb, cfg.MediatorSkipDB, cfg.MediatorCDROutDir, cfg.MediatorPseudoprepaid,
-		cfg.FreeswitchDirectionIdx, cfg.FreeswitchTORIdx, cfg.FreeswitchTenantIdx, cfg.FreeswitchSubjectIdx, cfg.FreeswitchAccountIdx,
-		cfg.FreeswitchDestIdx, cfg.FreeswitchTimeStartIdx, cfg.FreeswitchDurationIdx, cfg.FreeswitchUUIDIdx)
-	if err != nil {
-		rater.Logger.Crit(fmt.Sprintf("Mediator config parsing error: %v", err))
-		exitChan <- true
-	}
-
-	m.TrackCDRFiles(cfg.MediatorCDRInDir)
-}
-
-func startSessionManager(responder *rater.Responder, loggerDb rater.DataStorage) {
-	var connector rater.Connector
-	if cfg.SMRater == INTERNAL {
-		connector = responder
-	} else {
-		var client *rpc.Client
-		var err error
-		if cfg.SMRPCEncoding == JSON {
-			client, err = jsonrpc.Dial("tcp", cfg.SMRater)
-		} else {
-			client, err = rpc.Dial("tcp", cfg.SMRater)
-		}
-		if err != nil {
-			rater.Logger.Crit(fmt.Sprintf("Could not connect to rater: %v", err))
-			exitChan <- true
-		}
-		connector = &rater.RPCClientConnector{Client: client}
-	}
-	switch cfg.SMSwitchType {
-	case FS:
-		dp, _ := time.ParseDuration(fmt.Sprintf("%vs", cfg.SMDebitInterval))
-		sm = sessionmanager.NewFSSessionManager(loggerDb, connector, dp)
-		errConn := sm.Connect(cfg)
-		if errConn != nil {
-			rater.Logger.Err(fmt.Sprintf("<SessionManager> error: %s!", errConn))
-		}
-	default:
-		rater.Logger.Err(fmt.Sprintf("<SessionManager> Unsupported session manger type: %s!", cfg.SMSwitchType))
-		exitChan <- true
-	}
-	exitChan <- true
-}
-
-func checkConfigSanity() {
-	if cfg.SMEnabled && cfg.RaterEnabled && cfg.RaterBalancer != DISABLED {
-		rater.Logger.Crit("The session manager must not be enabled on a worker rater (change [rater]/balancer to disabled)!")
-		exitChan <- true
-	}
-	if cfg.BalancerEnabled && cfg.RaterEnabled && cfg.RaterBalancer != DISABLED {
-		rater.Logger.Crit("The balancer is enabled so it cannot connect to anatoher balancer (change [rater]/balancer to disabled)!")
-		exitChan <- true
-	}
-
-	// check if the session manager or mediator is connectting via loopback
-	// if they are using the same encoding as the rater/balancer
-	// this scenariou should be used for debug puropses only (it is racy anyway)
-	// and it might be forbidden in the future
-	if strings.Contains(cfg.SMRater, "localhost") || strings.Contains(cfg.SMRater, "127.0.0.1") {
-		if cfg.BalancerEnabled {
-			if cfg.BalancerRPCEncoding != cfg.SMRPCEncoding {
-				rater.Logger.Crit("If you are connecting the session manager via the loopback to the balancer use the same type of rpc encoding!")
-				exitChan <- true
-			}
-		}
-		if cfg.RaterEnabled {
-			if cfg.RaterRPCEncoding != cfg.SMRPCEncoding {
-				rater.Logger.Crit("If you are connecting the session manager via the loopback to the arter use the same type of rpc encoding!")
-				exitChan <- true
-			}
-		}
-	}
-	if strings.Contains(cfg.MediatorRater, "localhost") || strings.Contains(cfg.MediatorRater, "127.0.0.1") {
-		if cfg.BalancerEnabled {
-			if cfg.BalancerRPCEncoding != cfg.MediatorRPCEncoding {
-				rater.Logger.Crit("If you are connecting the mediator via the loopback to the balancer use the same type of rpc encoding!")
-				exitChan <- true
-			}
-		}
-		if cfg.RaterEnabled {
-			if cfg.RaterRPCEncoding != cfg.MediatorRPCEncoding {
-				rater.Logger.Crit("If you are connecting the mediator via the loopback to the arter use the same type of rpc encoding!")
-				exitChan <- true
-			}
-		}
-	}
-}
-
-func configureDatabase(db_type, host, port, name, user, pass string) (getter rater.DataStorage, err error) {
-	switch db_type {
-	case REDIS:
-		db_nb, err := strconv.Atoi(name)
-		if err != nil {
-			rater.Logger.Crit("Redis db name must be an integer!")
-			exitChan <- true
-		}
-		if port != "" {
-			host += ":" + port
-		}
-		getter, err = rater.NewRedisStorage(host, db_nb, pass)
-	case MONGO:
-		getter, err = rater.NewMongoStorage(host, port, name, user, pass)
-	case POSTGRES:
-		getter, err = rater.NewPostgresStorage(host, port, name, user, pass)
-	default:
-		err = errors.New("unknown db")
-		rater.Logger.Crit("Unknown db type, exiting!")
-		exitChan <- true
-	}
-
-	if err != nil {
-		rater.Logger.Crit(fmt.Sprintf("Could not connect to db: %v, exiting!", err))
-		exitChan <- true
-	}
-	return
-}
-
-func main() {
-	flag.Parse()
-	if *version {
-		fmt.Println("CGRateS " + rater.VERSION)
-		return
-	}
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
-	cfg, err = config.NewCGRConfig(cfgPath)
-	if err != nil {
-		rater.Logger.Crit(fmt.Sprintf("Could not parse config: %s exiting!", err))
-		return
-	}
-	// some consitency checks
-	go checkConfigSanity()
-
-	var getter, loggerDb rater.DataStorage
-	getter, err = configureDatabase(cfg.DataDBType, cfg.DataDBHost, cfg.DataDBPort, cfg.DataDBName, cfg.DataDBUser, cfg.DataDBPass)
-	if err != nil { // Cannot configure getter database, show stopper
-		rater.Logger.Crit(fmt.Sprintf("Could not configure database: %s exiting!", err))
-		return
-	}
-	defer getter.Close()
-	rater.SetDataStorage(getter)
-	if cfg.LogDBType == SAME {
-		loggerDb = getter
-	} else {
-		loggerDb, err = configureDatabase(cfg.LogDBType, cfg.LogDBHost, cfg.LogDBPort, cfg.LogDBName, cfg.LogDBUser, cfg.LogDBPass)
-	}
-	if err != nil { // Cannot configure logger database, show stopper
-		rater.Logger.Crit(fmt.Sprintf("Could not configure logger database: %s exiting!", err))
-		return
-	}
-	defer loggerDb.Close()
-	rater.SetStorageLogger(loggerDb)
-
-	if cfg.SMDebitInterval > 0 {
-		if dp, err := time.ParseDuration(fmt.Sprintf("%vs", cfg.SMDebitInterval)); err == nil {
-			rater.SetDebitPeriod(dp)
-		}
-	}
-
-	if cfg.RaterEnabled && cfg.RaterBalancer != DISABLED && !cfg.BalancerEnabled {
-		go registerToBalancer()
-		go stopRaterSingnalHandler()
-	}
-	responder := &rater.Responder{ExitChan: exitChan}
-	if cfg.RaterEnabled && !cfg.BalancerEnabled && cfg.RaterListen != INTERNAL {
-		rater.Logger.Info(fmt.Sprintf("Starting CGRateS Rater on %s.", cfg.RaterListen))
-		go listenToRPCRequests(responder, cfg.RaterListen, cfg.RaterRPCEncoding)
-	}
-	if cfg.BalancerEnabled {
-		rater.Logger.Info(fmt.Sprintf("Starting CGRateS Balancer on %s.", cfg.BalancerListen))
-		go stopBalancerSingnalHandler()
-		responder.Bal = bal
-		go listenToRPCRequests(responder, cfg.BalancerListen, cfg.BalancerRPCEncoding)
-		if cfg.RaterEnabled {
-			rater.Logger.Info("Starting internal rater.")
-			bal.AddClient("local", new(rater.ResponderWorker))
-		}
-	}
-
-	if cfg.SchedulerEnabled {
-		rater.Logger.Info("Starting CGRateS Scheduler.")
-		go func() {
-			sched := scheduler.NewScheduler()
-			go reloadSchedulerSingnalHandler(sched, getter)
-			sched.LoadActionTimings(getter)
-			sched.Loop()
-		}()
-	}
-
-	if cfg.SMEnabled {
-		rater.Logger.Info("Starting CGRateS SessionManager.")
-		go startSessionManager(responder, loggerDb)
-		// close all sessions on shutdown
-		go shutdownSessionmanagerSingnalHandler()
-	}
-
-	if cfg.MediatorEnabled {
-		rater.Logger.Info("Starting CGRateS Mediator.")
-		go startMediator(responder, loggerDb)
-	}
-	<-exitChan
-	rater.Logger.Info("Stopped all components. CGRateS shutdown!")
-}

cmd/cgr-tester/README.md

@@ -0,0 +1,3 @@
+* Stress testing tools
+
+All stress testing tools work with ../../data/tariffplans/fs_germany_prep1/. So start a rater and than issue a `cgr-loader -flushdb -verbose` in that directory.

cmd/cgr-tester/cgr-tester.go

@@ -0,0 +1,189 @@
+/*
+Rating system designed to be used in VoIP Carriers World
+Copyright (C) 2012-2015 ITsysCOM
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"os"
+	"runtime"
+	"runtime/pprof"
+	"time"
+
+	"github.com/cgrates/cgrates/config"
+	"github.com/cgrates/cgrates/engine"
+	"github.com/cgrates/cgrates/utils"
+)
+
+var (
+	cgrConfig, _    = config.NewDefaultCGRConfig()
+	cpuprofile      = flag.String("cpuprofile", "", "write cpu profile to file")
+	memprofile      = flag.String("memprofile", "", "write memory profile to this file")
+	runs            = flag.Int("runs", 10000, "stress cycle number")
+	parallel        = flag.Int("parallel", 0, "run n requests in parallel")
+	ratingdb_type   = flag.String("ratingdb_type", cgrConfig.TpDbType, "The type of the RatingDb database <redis>")
+	ratingdb_host   = flag.String("ratingdb_host", cgrConfig.TpDbHost, "The RatingDb host to connect to.")
+	ratingdb_port   = flag.String("ratingdb_port", cgrConfig.TpDbPort, "The RatingDb port to bind to.")
+	ratingdb_name   = flag.String("ratingdb_name", cgrConfig.TpDbName, "The name/number of the RatingDb to connect to.")
+	ratingdb_user   = flag.String("ratingdb_user", cgrConfig.TpDbUser, "The RatingDb user to sign in as.")
+	ratingdb_pass   = flag.String("ratingdb_passwd", cgrConfig.TpDbPass, "The RatingDb user's password.")
+	accountdb_type  = flag.String("accountdb_type", cgrConfig.DataDbType, "The type of the AccountingDb database <redis>")
+	accountdb_host  = flag.String("accountdb_host", cgrConfig.DataDbHost, "The AccountingDb host to connect to.")
+	accountdb_port  = flag.String("accountdb_port", cgrConfig.DataDbPort, "The AccountingDb port to bind to.")
+	accountdb_name  = flag.String("accountdb_name", cgrConfig.DataDbName, "The name/number of the AccountingDb to connect to.")
+	accountdb_user  = flag.String("accountdb_user", cgrConfig.DataDbUser, "The AccountingDb user to sign in as.")
+	accountdb_pass  = flag.String("accountdb_passwd", cgrConfig.DataDbPass, "The AccountingDb user's password.")
+	dbdata_encoding = flag.String("dbdata_encoding", cgrConfig.DBDataEncoding, "The encoding used to store object data in strings.")
+	raterAddress    = flag.String("rater_address", "", "Rater address for remote tests. Empty for internal rater.")
+	tor             = flag.String("tor", utils.VOICE, "The type of record to use in queries.")
+	category        = flag.String("category", "call", "The Record category to test.")
+	tenant          = flag.String("tenant", "cgrates.org", "The type of record to use in queries.")
+	subject         = flag.String("subject", "1001", "The rating subject to use in queries.")
+	destination     = flag.String("destination", "1002", "The destination to use in queries.")
+	json            = flag.Bool("json", false, "Use JSON RPC")
+
+	nilDuration = time.Duration(0)
+)
+
+func durInternalRater(cd *engine.CallDescriptor) (time.Duration, error) {
+	ratingDb, err := engine.ConfigureRatingStorage(*ratingdb_type, *ratingdb_host, *ratingdb_port, *ratingdb_name, *ratingdb_user, *ratingdb_pass, *dbdata_encoding)
+	if err != nil {
+		return nilDuration, fmt.Errorf("Could not connect to rating database: %s", err.Error())
+	}
+	defer ratingDb.Close()
+	engine.SetRatingStorage(ratingDb)
+	accountDb, err := engine.ConfigureAccountingStorage(*accountdb_type, *accountdb_host, *accountdb_port, *accountdb_name, *accountdb_user, *accountdb_pass, *dbdata_encoding)
+	if err != nil {
+		return nilDuration, fmt.Errorf("Could not connect to accounting database: %s", err.Error())
+	}
+	defer accountDb.Close()
+	engine.SetAccountingStorage(accountDb)
+	if err := ratingDb.CacheRatingAll(); err != nil {
+		return nilDuration, fmt.Errorf("Cache rating error: %s", err.Error())
+	}
+	if err := accountDb.CacheAccountingAll(); err != nil {
+		return nilDuration, fmt.Errorf("Cache accounting error: %s", err.Error())
+	}
+	log.Printf("Runnning %d cycles...", *runs)
+	var result *engine.CallCost
+	j := 0
+	start := time.Now()
+	for i := 0; i < *runs; i++ {
+		result, err = cd.GetCost()
+		if *memprofile != "" {
+			runtime.MemProfileRate = 1
+			runtime.GC()
+			f, err := os.Create(*memprofile)
+			if err != nil {
+				log.Fatal(err)
+			}
+			pprof.WriteHeapProfile(f)
+			f.Close()
+			break
+		}
+		j = i
+	}
+	log.Print(result, j, err)
+	memstats := new(runtime.MemStats)
+	runtime.ReadMemStats(memstats)
+	log.Printf("memstats before GC: Kbytes = %d footprint = %d",
+		memstats.HeapAlloc/1024, memstats.Sys/1024)
+	return time.Since(start), nil
+}
+
+func durRemoteRater(cd *engine.CallDescriptor) (time.Duration, error) {
+	result := engine.CallCost{}
+	var client *rpc.Client
+	var err error
+	if *json {
+		client, err = jsonrpc.Dial("tcp", *raterAddress)
+	} else {
+		client, err = rpc.Dial("tcp", *raterAddress)
+	}
+
+	if err != nil {
+		return nilDuration, fmt.Errorf("Could not connect to engine: %s", err.Error())
+	}
+	defer client.Close()
+	start := time.Now()
+	if *parallel > 0 {
+		// var divCall *rpc.Call
+		var sem = make(chan int, *parallel)
+		var finish = make(chan int)
+		for i := 0; i < *runs; i++ {
+			go func() {
+				sem <- 1
+				client.Call("Responder.GetCost", cd, &result)
+				<-sem
+				finish <- 1
+				// divCall = client.Go("Responder.GetCost", cd, &result, nil)
+			}()
+		}
+		for i := 0; i < *runs; i++ {
+			<-finish
+		}
+		// <-divCall.Done
+	} else {
+		for j := 0; j < *runs; j++ {
+			client.Call("Responder.GetCost", cd, &result)
+		}
+	}
+	log.Println(result)
+	return time.Since(start), nil
+}
+
+func main() {
+	flag.Parse()
+	runtime.GOMAXPROCS(runtime.NumCPU() - 1)
+
+	if *cpuprofile != "" {
+		f, err := os.Create(*cpuprofile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+	cd := &engine.CallDescriptor{
+		TimeStart:     time.Date(2014, time.December, 11, 55, 30, 0, 0, time.UTC),
+		TimeEnd:       time.Date(2014, time.December, 11, 55, 31, 0, 0, time.UTC),
+		DurationIndex: 60 * time.Second,
+		Direction:     "*out",
+		TOR:           *tor,
+		Category:      *category,
+		Tenant:        *tenant,
+		Subject:       *subject,
+		Destination:   *destination,
+	}
+	var duration time.Duration
+	var err error
+	if len(*raterAddress) == 0 {
+		duration, err = durInternalRater(cd)
+	} else {
+		duration, err = durRemoteRater(cd)
+	}
+	if err != nil {
+		log.Fatal(err.Error())
+	} else {
+		log.Printf("Elapsed: %d resulted: %f req/s.", duration, float64(*runs)/duration.Seconds())
+	}
+}

cmd/stress/cgr-raterstress/cgr-raterstress.go

@@ -1,84 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package main
-
-import (
-	"flag"
-	"github.com/cgrates/cgrates/rater"
-	"log"
-	"net/rpc"
-	//"net/rpc/jsonrpc"
-	"net/rpc/jsonrpc"
-	"runtime"
-	"time"
-)
-
-var (
-	balancer = flag.String("balancer", "localhost:2001", "balancer server address")
-	runs     = flag.Int("runs", 10000, "stress cycle number")
-	parallel = flag.Int("parallel", 0, "run n requests in parallel")
-	json     = flag.Bool("json", false, "use JSON for RPC encoding")
-)
-
-func main() {
-	flag.Parse()
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
-	t1 := time.Date(2012, time.February, 02, 17, 30, 0, 0, time.UTC)
-	t2 := time.Date(2012, time.February, 02, 18, 30, 0, 0, time.UTC)
-	cd := rater.CallDescriptor{Direction: "OUT", TOR: "0", Tenant: "vdf", Subject: "rif", Destination: "0256", TimeStart: t1, TimeEnd: t2}
-	result := rater.CallCost{}
-	var client *rpc.Client
-	var err error
-	if *json {
-		client, err = jsonrpc.Dial("tcp", *balancer)
-	} else {
-		client, err = rpc.Dial("tcp", *balancer)
-	}
-	if err != nil {
-		log.Fatal("Could not connect to rater: ", err)
-	}
-	start := time.Now()
-	if *parallel > 0 {
-		// var divCall *rpc.Call
-		var sem = make(chan int, *parallel)
-		var finish = make(chan int)
-		for i := 0; i < *runs; i++ {
-			go func() {
-				sem <- 1
-				client.Call("Responder.GetCost", cd, &result)
-				<-sem
-				finish <- 1
-				// divCall = client.Go("Responder.GetCost", cd, &result, nil)	
-			}()
-		}
-		for i := 0; i < *runs; i++ {
-			<-finish
-		}
-		// <-divCall.Done
-	} else {
-		for j := 0; j < *runs; j++ {
-			client.Call("Responder.GetCost", cd, &result)
-		}
-	}
-	duration := time.Since(start)
-	log.Println(result)
-	client.Close()
-	log.Printf("Elapsed: %v resulted: %v req/s.", duration, float64(*runs)/duration.Seconds())
-}

cmd/stress/cgr-spansstress/cgr-spansstress.go

@@ -1,83 +0,0 @@
-/*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>
-*/
-
-package main
-
-import (
-	"flag"
-	"github.com/cgrates/cgrates/rater"
-	"log"
-	"os"
-	"runtime"
-	"runtime/pprof"
-	"time"
-)
-
-var (
-	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
-	memprofile = flag.String("memprofile", "", "write memory profile to this file")
-	runs       = flag.Int("runs", 10000, "stress cycle number")
-)
-
-func main() {
-	flag.Parse()
-	runtime.GOMAXPROCS(runtime.NumCPU() - 1)
-
-	if *cpuprofile != "" {
-		f, err := os.Create(*cpuprofile)
-		if err != nil {
-			log.Fatal(err)
-		}
-		pprof.StartCPUProfile(f)
-		defer pprof.StopCPUProfile()
-	}
-	if *memprofile != "" {
-		f, err := os.Create(*memprofile)
-		if err != nil {
-			log.Fatal(err)
-		}
-		pprof.WriteHeapProfile(f)
-		f.Close()
-		return
-	}
-	t1 := time.Date(2012, time.February, 02, 17, 30, 0, 0, time.UTC)
-	t2 := time.Date(2012, time.February, 02, 18, 30, 0, 0, time.UTC)
-	cd := rater.CallDescriptor{Direction: "OUT", TOR: "0", Tenant: "vdf", Subject: "rif", Destination: "0256", TimeStart: t1, TimeEnd: t2}
-
-	getter, err := rater.NewRedisStorage("localhost:6379", 10, "")
-	//getter, err := rater.NewMongoStorage("localhost", "cgrates")
-	defer getter.Close()
-
-	rater.SetDataStorage(getter)
-
-	log.Printf("Runnning %d cycles...", *runs)
-	var result *rater.CallCost
-	j := 0
-	start := time.Now()
-	for i := 0; i < *runs; i++ {
-		result, err = cd.GetCost()
-		j = i
-	}
-	duration := time.Since(start)
-	log.Print(result, j, err)
-	memstats := new(runtime.MemStats)
-	runtime.ReadMemStats(memstats)
-	log.Printf("memstats before GC: Kbytes = %d footprint = %d",
-		memstats.HeapAlloc/1024, memstats.Sys/1024)
-	log.Printf("Elapsed: %v resulted: %v req/s.", duration, float64(*runs)/duration.Seconds())
-}

config/cdrcconfig.go

@@ -0,0 +1,151 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"time"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+type CdrcConfig struct {
+	Enabled                 bool            // Enable/Disable the profile
+	DryRun                  bool            // Do not post CDRs to the server
+	Cdrs                    string          // The address where CDRs can be reached
+	CdrFormat               string          // The type of CDR file to process <csv|opensips_flatstore>
+	FieldSeparator          rune            // The separator to use when reading csvs
+	DataUsageMultiplyFactor float64         // Conversion factor for data usage
+	Timezone                string          // timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB>
+	RunDelay                time.Duration   // Delay between runs, 0 for inotify driven requests
+	MaxOpenFiles            int             // Maximum number of files opened simultaneously
+	CdrInDir                string          // Folder to process CDRs from
+	CdrOutDir               string          // Folder to move processed CDRs to
+	FailedCallsPrefix       string          // Used in case of flatstore CDRs to avoid searching for BYE records
+	CdrSourceId             string          // Source identifier for the processed CDRs
+	CdrFilter               utils.RSRFields // Filter CDR records to import
+	PartialRecordCache      time.Duration   // Duration to cache partial records when not pairing
+	HeaderFields            []*CfgCdrField
+	ContentFields           []*CfgCdrField
+	TrailerFields           []*CfgCdrField
+}
+
+func (self *CdrcConfig) loadFromJsonCfg(jsnCfg *CdrcJsonCfg) error {
+	if jsnCfg == nil {
+		return nil
+	}
+	var err error
+	if jsnCfg.Enabled != nil {
+		self.Enabled = *jsnCfg.Enabled
+	}
+	if jsnCfg.Dry_run != nil {
+		self.DryRun = *jsnCfg.Dry_run
+	}
+	if jsnCfg.Cdrs != nil {
+		self.Cdrs = *jsnCfg.Cdrs
+	}
+	if jsnCfg.Cdr_format != nil {
+		self.CdrFormat = *jsnCfg.Cdr_format
+	}
+	if jsnCfg.Field_separator != nil && len(*jsnCfg.Field_separator) > 0 {
+		sepStr := *jsnCfg.Field_separator
+		self.FieldSeparator = rune(sepStr[0])
+	}
+	if jsnCfg.Data_usage_multiply_factor != nil {
+		self.DataUsageMultiplyFactor = *jsnCfg.Data_usage_multiply_factor
+	}
+	if jsnCfg.Timezone != nil {
+		self.Timezone = *jsnCfg.Timezone
+	}
+	if jsnCfg.Run_delay != nil {
+		self.RunDelay = time.Duration(*jsnCfg.Run_delay) * time.Second
+	}
+	if jsnCfg.Max_open_files != nil {
+		self.MaxOpenFiles = *jsnCfg.Max_open_files
+	}
+	if jsnCfg.Cdr_in_dir != nil {
+		self.CdrInDir = *jsnCfg.Cdr_in_dir
+	}
+	if jsnCfg.Cdr_out_dir != nil {
+		self.CdrOutDir = *jsnCfg.Cdr_out_dir
+	}
+	if jsnCfg.Failed_calls_prefix != nil {
+		self.FailedCallsPrefix = *jsnCfg.Failed_calls_prefix
+	}
+	if jsnCfg.Cdr_source_id != nil {
+		self.CdrSourceId = *jsnCfg.Cdr_source_id
+	}
+	if jsnCfg.Cdr_filter != nil {
+		if self.CdrFilter, err = utils.ParseRSRFields(*jsnCfg.Cdr_filter, utils.INFIELD_SEP); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Partial_record_cache != nil {
+		if self.PartialRecordCache, err = utils.ParseDurationWithSecs(*jsnCfg.Partial_record_cache); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Header_fields != nil {
+		if self.HeaderFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Header_fields); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Content_fields != nil {
+		if self.ContentFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Content_fields); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Trailer_fields != nil {
+		if self.TrailerFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Trailer_fields); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Clone itself into a new CdrcConfig
+func (self *CdrcConfig) Clone() *CdrcConfig {
+	clnCdrc := new(CdrcConfig)
+	clnCdrc.Enabled = self.Enabled
+	clnCdrc.Cdrs = self.Cdrs
+	clnCdrc.CdrFormat = self.CdrFormat
+	clnCdrc.FieldSeparator = self.FieldSeparator
+	clnCdrc.DataUsageMultiplyFactor = self.DataUsageMultiplyFactor
+	clnCdrc.Timezone = self.Timezone
+	clnCdrc.RunDelay = self.RunDelay
+	clnCdrc.MaxOpenFiles = self.MaxOpenFiles
+	clnCdrc.CdrInDir = self.CdrInDir
+	clnCdrc.CdrOutDir = self.CdrOutDir
+	clnCdrc.CdrSourceId = self.CdrSourceId
+	clnCdrc.HeaderFields = make([]*CfgCdrField, len(self.HeaderFields))
+	clnCdrc.ContentFields = make([]*CfgCdrField, len(self.ContentFields))
+	clnCdrc.TrailerFields = make([]*CfgCdrField, len(self.TrailerFields))
+	for idx, fld := range self.HeaderFields {
+		clonedVal := *fld
+		clnCdrc.HeaderFields[idx] = &clonedVal
+	}
+	for idx, fld := range self.ContentFields {
+		clonedVal := *fld
+		clnCdrc.ContentFields[idx] = &clonedVal
+	}
+	for idx, fld := range self.TrailerFields {
+		clonedVal := *fld
+		clnCdrc.TrailerFields[idx] = &clonedVal
+	}
+	return clnCdrc
+}

config/cdrcconfig_test.go

@@ -0,0 +1,19 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config

config/cdreconfig.go

@@ -0,0 +1,126 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+// One instance of CdrExporter
+type CdreConfig struct {
+	CdrFormat                  string
+	FieldSeparator             rune
+	DataUsageMultiplyFactor    float64
+	SmsUsageMultiplyFactor     float64
+	GenericUsageMultiplyFactor float64
+	CostMultiplyFactor         float64
+	CostRoundingDecimals       int
+	CostShiftDigits            int
+	MaskDestId                 string
+	MaskLength                 int
+	ExportDir                  string
+	HeaderFields               []*CfgCdrField
+	ContentFields              []*CfgCdrField
+	TrailerFields              []*CfgCdrField
+}
+
+func (self *CdreConfig) loadFromJsonCfg(jsnCfg *CdreJsonCfg) error {
+	if jsnCfg == nil {
+		return nil
+	}
+	var err error
+	if jsnCfg.Cdr_format != nil {
+		self.CdrFormat = *jsnCfg.Cdr_format
+	}
+	if jsnCfg.Field_separator != nil && len(*jsnCfg.Field_separator) > 0 { // Make sure we got at least one character so we don't get panic here
+		sepStr := *jsnCfg.Field_separator
+		self.FieldSeparator = rune(sepStr[0])
+	}
+	if jsnCfg.Data_usage_multiply_factor != nil {
+		self.DataUsageMultiplyFactor = *jsnCfg.Data_usage_multiply_factor
+	}
+	if jsnCfg.Sms_usage_multiply_factor != nil {
+		self.SmsUsageMultiplyFactor = *jsnCfg.Sms_usage_multiply_factor
+	}
+	if jsnCfg.Generic_usage_multiply_factor != nil {
+		self.GenericUsageMultiplyFactor = *jsnCfg.Generic_usage_multiply_factor
+	}
+	if jsnCfg.Cost_multiply_factor != nil {
+		self.CostMultiplyFactor = *jsnCfg.Cost_multiply_factor
+	}
+	if jsnCfg.Cost_rounding_decimals != nil {
+		self.CostRoundingDecimals = *jsnCfg.Cost_rounding_decimals
+	}
+	if jsnCfg.Cost_shift_digits != nil {
+		self.CostShiftDigits = *jsnCfg.Cost_shift_digits
+	}
+	if jsnCfg.Mask_destination_id != nil {
+		self.MaskDestId = *jsnCfg.Mask_destination_id
+	}
+	if jsnCfg.Mask_length != nil {
+		self.MaskLength = *jsnCfg.Mask_length
+	}
+	if jsnCfg.Export_dir != nil {
+		self.ExportDir = *jsnCfg.Export_dir
+	}
+	if jsnCfg.Header_fields != nil {
+		if self.HeaderFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Header_fields); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Content_fields != nil {
+		if self.ContentFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Content_fields); err != nil {
+			return err
+		}
+	}
+	if jsnCfg.Trailer_fields != nil {
+		if self.TrailerFields, err = CfgCdrFieldsFromCdrFieldsJsonCfg(*jsnCfg.Trailer_fields); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Clone itself into a new CdreConfig
+func (self *CdreConfig) Clone() *CdreConfig {
+	clnCdre := new(CdreConfig)
+	clnCdre.CdrFormat = self.CdrFormat
+	clnCdre.FieldSeparator = self.FieldSeparator
+	clnCdre.DataUsageMultiplyFactor = self.DataUsageMultiplyFactor
+	clnCdre.SmsUsageMultiplyFactor = self.SmsUsageMultiplyFactor
+	clnCdre.GenericUsageMultiplyFactor = self.GenericUsageMultiplyFactor
+	clnCdre.CostMultiplyFactor = self.CostMultiplyFactor
+	clnCdre.CostRoundingDecimals = self.CostRoundingDecimals
+	clnCdre.CostShiftDigits = self.CostShiftDigits
+	clnCdre.MaskDestId = self.MaskDestId
+	clnCdre.MaskLength = self.MaskLength
+	clnCdre.ExportDir = self.ExportDir
+	clnCdre.HeaderFields = make([]*CfgCdrField, len(self.HeaderFields))
+	for idx, fld := range self.HeaderFields {
+		clonedVal := *fld
+		clnCdre.HeaderFields[idx] = &clonedVal
+	}
+	clnCdre.ContentFields = make([]*CfgCdrField, len(self.ContentFields))
+	for idx, fld := range self.ContentFields {
+		clonedVal := *fld
+		clnCdre.ContentFields[idx] = &clonedVal
+	}
+	clnCdre.TrailerFields = make([]*CfgCdrField, len(self.TrailerFields))
+	for idx, fld := range self.TrailerFields {
+		clonedVal := *fld
+		clnCdre.TrailerFields[idx] = &clonedVal
+	}
+	return clnCdre
+}

config/cdreconfig_test.go

@@ -0,0 +1,97 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+package config
+
+import (
+	"github.com/cgrates/cgrates/utils"
+	"reflect"
+	"testing"
+)
+
+func TestCdreCfgClone(t *testing.T) {
+	cgrIdRsrs, _ := utils.ParseRSRFields("cgrid", utils.INFIELD_SEP)
+	runIdRsrs, _ := utils.ParseRSRFields("mediation_runid", utils.INFIELD_SEP)
+	emptyFields := []*CfgCdrField{}
+	initContentFlds := []*CfgCdrField{
+		&CfgCdrField{Tag: "CgrId",
+			Type:       "cdrfield",
+			CdrFieldId: "cgrid",
+			Value:      cgrIdRsrs},
+		&CfgCdrField{Tag: "RunId",
+			Type:       "cdrfield",
+			CdrFieldId: "mediation_runid",
+			Value:      runIdRsrs},
+	}
+	initCdreCfg := &CdreConfig{
+		CdrFormat:               "csv",
+		FieldSeparator:          rune(','),
+		DataUsageMultiplyFactor: 1.0,
+		CostMultiplyFactor:      1.0,
+		CostRoundingDecimals:    -1,
+		CostShiftDigits:         0,
+		MaskDestId:              "MASKED_DESTINATIONS",
+		MaskLength:              0,
+		ExportDir:               "/var/log/cgrates/cdre",
+		ContentFields:           initContentFlds,
+	}
+	eClnContentFlds := []*CfgCdrField{
+		&CfgCdrField{Tag: "CgrId",
+			Type:       "cdrfield",
+			CdrFieldId: "cgrid",
+			Value:      cgrIdRsrs},
+		&CfgCdrField{Tag: "RunId",
+			Type:       "cdrfield",
+			CdrFieldId: "mediation_runid",
+			Value:      runIdRsrs},
+	}
+	eClnCdreCfg := &CdreConfig{
+		CdrFormat:               "csv",
+		FieldSeparator:          rune(','),
+		DataUsageMultiplyFactor: 1.0,
+		CostMultiplyFactor:      1.0,
+		CostRoundingDecimals:    -1,
+		CostShiftDigits:         0,
+		MaskDestId:              "MASKED_DESTINATIONS",
+		MaskLength:              0,
+		ExportDir:               "/var/log/cgrates/cdre",
+		HeaderFields:            emptyFields,
+		ContentFields:           eClnContentFlds,
+		TrailerFields:           emptyFields,
+	}
+	clnCdreCfg := initCdreCfg.Clone()
+	if !reflect.DeepEqual(eClnCdreCfg, clnCdreCfg) {
+		t.Errorf("Cloned result: %+v", clnCdreCfg)
+	}
+	initCdreCfg.DataUsageMultiplyFactor = 1024.0
+	if !reflect.DeepEqual(eClnCdreCfg, clnCdreCfg) { // MOdifying a field after clone should not affect cloned instance
+		t.Errorf("Cloned result: %+v", clnCdreCfg)
+	}
+	initContentFlds[0].Tag = "Destination"
+	if !reflect.DeepEqual(eClnCdreCfg, clnCdreCfg) { // MOdifying a field after clone should not affect cloned instance
+		t.Errorf("Cloned result: %+v", clnCdreCfg)
+	}
+	clnCdreCfg.CostShiftDigits = 2
+	if initCdreCfg.CostShiftDigits != 0 {
+		t.Error("Unexpected CostShiftDigits: ", initCdreCfg.CostShiftDigits)
+	}
+	clnCdreCfg.ContentFields[0].CdrFieldId = "destination"
+	if initCdreCfg.ContentFields[0].CdrFieldId != "cgrid" {
+		t.Error("Unexpected change of CdrFieldId: ", initCdreCfg.ContentFields[0].CdrFieldId)
+	}
+
+}

config/cdrstatsconfig.go

@@ -0,0 +1,49 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"time"
+)
+
+type CdrStatsConfig struct {
+	Id               string        // Config id, unique per config instance
+	QueueLength      int           // Number of items in the stats buffer
+	TimeWindow       time.Duration // Will only keep the CDRs who's call setup time is not older than time.Now()-TimeWindow
+	SaveInterval     time.Duration
+	Metrics          []string    // ASR, ACD, ACC
+	SetupInterval    []time.Time // 2 or less items (>= start interval,< stop_interval)
+	TORs             []string
+	CdrHosts         []string
+	CdrSources       []string
+	ReqTypes         []string
+	Directions       []string
+	Tenants          []string
+	Categories       []string
+	Accounts         []string
+	Subjects         []string
+	DestinationIds   []string
+	UsageInterval    []time.Duration // 2 or less items (>= Usage, <Usage)
+	Suppliers        []string
+	DisconnectCauses []string
+	MediationRunIds  []string
+	RatedAccounts    []string
+	RatedSubjects    []string
+	CostInterval     []float64 // 2 or less items, (>=Cost, <Cost)
+}

config/cfg_data.json

@@ -0,0 +1,51 @@
+{
+
+// Real-time Charging System for Telecom & ISP environments
+// Copyright (C) ITsysCOM GmbH
+//
+// This file contains the default configuration hardcoded into CGRateS.
+// This is what you get when you load CGRateS with an empty configuration file.
+
+"general": {
+	"default_reqtype": "*pseudoprepaid",				// default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated>
+},
+
+"cdrs": {
+	"enabled": true,						// start the CDR Server service:  <true|false>
+},
+
+"rater": {
+	"enabled": true,						// enable Rater service: <true|false>
+},
+
+"cdrc": {
+	"CDRC-CSV1": {
+		"enabled": true,							// enable CDR client functionality
+		"cdr_in_dir": "/tmp/cgrates/cdrc1/in",		// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/tmp/cgrates/cdrc1/out",	// absolute path towards the directory where processed CDRs will be moved
+		"cdr_source_id": "csv1",					// free form field, tag identifying the source of the CDRs within CDRS database
+	},
+	"CDRC-CSV2": {
+		"enabled": true,							// enable CDR client functionality
+		"cdr_in_dir": "/tmp/cgrates/cdrc2/in",		// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/tmp/cgrates/cdrc2/out",	// absolute path towards the directory where processed CDRs will be moved
+		"data_usage_multiply_factor": 0.000976563,
+		"run_delay": 1,
+		"cdr_source_id": "csv2",					// free form field, tag identifying the source of the CDRs within CDRS database
+		"content_fields":[							// import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
+			{"cdr_field_id": "tor", "value": "~7:s/^(voice|data|sms|generic)$/*$1/"},
+			{"cdr_field_id": "answer_time", "value": "1"},
+			{"cdr_field_id": "usage", "value": "~9:s/^(\\d+)$/${1}s/"},
+		],
+	},
+},
+
+"sm_freeswitch": {
+	"enabled": true,				// starts SessionManager service: <true|false>
+	"connections":[					// instantiate connections to multiple FreeSWITCH servers
+		{"server": "1.2.3.4:8021", "password": "ClueCon", "reconnects": 5},
+		{"server": "2.3.4.5:8021", "password": "ClueCon", "reconnects": 5},
+	],
+},
+
+}

config/cfg_data2.json

@@ -0,0 +1,30 @@
+{
+
+"cdrc": {
+	"CDRC-CSV2": {
+		"enabled": true,							// enable CDR client functionality
+		"cdr_in_dir": "/tmp/cgrates/cdrc2/in",		// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/tmp/cgrates/cdrc2/out",	// absolute path towards the directory where processed CDRs will be moved
+		"data_usage_multiply_factor": 0.000976563,
+		"cdr_source_id": "csv2",					// free form field, tag identifying the source of the CDRs within CDRS database
+		"content_fields":[								// import template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
+			{"cdr_field_id": "tor", "value": "~7:s/^(voice|data|sms|generic)$/*$1/"},
+			{"cdr_field_id": "answer_time", "value": "2"},
+		],
+	},
+	"CDRC-CSV3": {
+		"enabled": true,							// enable CDR client functionality
+		"cdr_in_dir": "/tmp/cgrates/cdrc3/in",		// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/tmp/cgrates/cdrc3/out",	// absolute path towards the directory where processed CDRs will be moved
+		"cdr_source_id": "csv3",					// free form field, tag identifying the source of the CDRs within CDRS database
+	},
+},
+
+"sm_freeswitch": {
+	"enabled": true,				// starts SessionManager service: <true|false>
+	"connections":[					// instantiate connections to multiple FreeSWITCH servers
+		{"server": "2.3.4.5:8021", "password": "ClueCon", "reconnects": 5},
+	],
+},
+
+}

config/cfgcdrfield.go

@@ -0,0 +1,92 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"github.com/cgrates/cgrates/utils"
+)
+
+func NewCfgCdrFieldFromCdrFieldJsonCfg(jsnCfgFld *CdrFieldJsonCfg) (*CfgCdrField, error) {
+	var err error
+	cfgFld := new(CfgCdrField)
+	if jsnCfgFld.Tag != nil {
+		cfgFld.Tag = *jsnCfgFld.Tag
+	}
+	if jsnCfgFld.Type != nil {
+		cfgFld.Type = *jsnCfgFld.Type
+	}
+	if jsnCfgFld.Cdr_field_id != nil {
+		cfgFld.CdrFieldId = *jsnCfgFld.Cdr_field_id
+	}
+	if jsnCfgFld.Metatag_id != nil {
+		cfgFld.MetatagId = *jsnCfgFld.Metatag_id
+	}
+	if jsnCfgFld.Value != nil {
+		if cfgFld.Value, err = utils.ParseRSRFields(*jsnCfgFld.Value, utils.INFIELD_SEP); err != nil {
+			return nil, err
+		}
+	}
+	if jsnCfgFld.Field_filter != nil {
+		if cfgFld.FieldFilter, err = utils.ParseRSRFields(*jsnCfgFld.Field_filter, utils.INFIELD_SEP); err != nil {
+			return nil, err
+		}
+	}
+	if jsnCfgFld.Width != nil {
+		cfgFld.Width = *jsnCfgFld.Width
+	}
+	if jsnCfgFld.Strip != nil {
+		cfgFld.Strip = *jsnCfgFld.Strip
+	}
+	if jsnCfgFld.Padding != nil {
+		cfgFld.Padding = *jsnCfgFld.Padding
+	}
+	if jsnCfgFld.Layout != nil {
+		cfgFld.Layout = *jsnCfgFld.Layout
+	}
+	if jsnCfgFld.Mandatory != nil {
+		cfgFld.Mandatory = *jsnCfgFld.Mandatory
+	}
+	return cfgFld, nil
+}
+
+type CfgCdrField struct {
+	Tag         string // Identifier for the administrator
+	Type        string // Type of field
+	CdrFieldId  string // StoredCdr field name
+	MetatagId   string
+	Value       utils.RSRFields
+	FieldFilter utils.RSRFields
+	Width       int
+	Strip       string
+	Padding     string
+	Layout      string
+	Mandatory   bool
+}
+
+func CfgCdrFieldsFromCdrFieldsJsonCfg(jsnCfgFldss []*CdrFieldJsonCfg) ([]*CfgCdrField, error) {
+	retFields := make([]*CfgCdrField, len(jsnCfgFldss))
+	for idx, jsnFld := range jsnCfgFldss {
+		if cfgFld, err := NewCfgCdrFieldFromCdrFieldJsonCfg(jsnFld); err != nil {
+			return nil, err
+		} else {
+			retFields[idx] = cfgFld
+		}
+	}
+	return retFields, nil
+}

config/cfgcdrfield_test.go

@@ -0,0 +1,19 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config

config/config_defaults.go

@@ -0,0 +1,281 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+const CGRATES_CFG_JSON = `
+{
+
+// Real-time Charging System for Telecom & ISP environments
+// Copyright (C) ITsysCOM GmbH
+//
+// This file contains the default configuration hardcoded into CGRateS.
+// This is what you get when you load CGRateS with an empty configuration file.
+
+"general": {
+	"http_skip_tls_verify": false,			// if enabled Http Client will accept any TLS certificate
+	"rounding_decimals": 10,				// system level precision for floats
+	"dbdata_encoding": "msgpack",			// encoding used to store object data in strings: <msgpack|json>
+	"tpexport_dir": "/var/log/cgrates/tpe",	// path towards export folder for offline Tariff Plans
+	"default_reqtype": "*rated",			// default request type to consider when missing from requests: <""|*prepaid|*postpaid|*pseudoprepaid|*rated>
+	"default_category": "call",				// default Type of Record to consider when missing from requests
+	"default_tenant": "cgrates.org",		// default Tenant to consider when missing from requests
+	"default_subject": "cgrates",			// default rating Subject to consider when missing from requests
+	"default_timezone": "Local",			// default timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB>
+	"connect_attempts": 3,					// initial server connect attempts
+	"response_cache_ttl": "3s",				// the life span of a cached response
+	"reconnects": -1,						// number of retries in case of connection lost
+	"internal_ttl": "2m",					// maximum duration to wait for internal connections before giving up
+},
+
+
+"listen": {
+	"rpc_json": "127.0.0.1:2012",			// RPC JSON listening address
+	"rpc_gob": "127.0.0.1:2013",			// RPC GOB listening address
+	"http": "127.0.0.1:2080",				// HTTP listening address
+},
+
+
+"tariffplan_db": {							// database used to store active tariff plan configuration
+	"db_type": "redis",						// tariffplan_db type: <redis>
+	"db_host": "127.0.0.1",					// tariffplan_db host address
+	"db_port": 6379, 						// port to reach the tariffplan_db
+	"db_name": "10", 						// tariffplan_db name to connect to
+	"db_user": "", 							// sername to use when connecting to tariffplan_db
+	"db_passwd": "", 						// password to use when connecting to tariffplan_db
+},
+
+
+"data_db": {								// database used to store runtime data (eg: accounts, cdr stats)
+	"db_type": "redis",						// data_db type: <redis>
+	"db_host": "127.0.0.1",					// data_db host address
+	"db_port": 6379, 						// data_db port to reach the database
+	"db_name": "11", 						// data_db database name to connect to
+	"db_user": "", 							// username to use when connecting to data_db
+	"db_passwd": "", 						// password to use when connecting to data_db
+	"load_history_size": 10,				// Number of records in the load history
+},
+
+
+"stor_db": {								// database used to store offline tariff plans and CDRs
+	"db_type": "mysql",						// stor database type to use: <mysql|postgres>
+	"db_host": "127.0.0.1",					// the host to connect to
+	"db_port": 3306,						// the port to reach the stordb
+	"db_name": "cgrates",					// stor database name
+	"db_user": "cgrates",					// username to use when connecting to stordb
+	"db_passwd": "CGRateS.org",				// password to use when connecting to stordb
+	"max_open_conns": 100,					// maximum database connections opened
+	"max_idle_conns": 10,					// maximum database connections idle
+},
+
+
+"balancer": {
+	"enabled": false,						// start Balancer service: <true|false>
+},
+
+
+"rater": {
+	"enabled": false,						// enable Rater service: <true|false>
+	"balancer": "",							// register to balancer as worker: <""|internal|x.y.z.y:1234>
+	"cdrstats": "",							// address where to reach the cdrstats service, empty to disable stats functionality: <""|internal|x.y.z.y:1234>
+	"historys": "",							// address where to reach the history service, empty to disable history functionality: <""|internal|x.y.z.y:1234>
+	"pubsubs": "",							// address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234>
+	"users": "",							// address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234>
+	"aliases": "",							// address where to reach the aliases service, empty to disable aliases functionality: <""|internal|x.y.z.y:1234>
+},
+
+
+"scheduler": {
+	"enabled": false,						// start Scheduler service: <true|false>
+},
+
+
+"cdrs": {
+	"enabled": false,						// start the CDR Server service:  <true|false>
+	"extra_fields": [],						// extra fields to store in CDRs for non-generic CDRs
+	"store_cdrs": true,						// store cdrs in storDb
+	"rater": "internal",					// address where to reach the Rater for cost calculation, empty to disable functionality: <""|internal|x.y.z.y:1234>
+	"pubsubs": "",							// address where to reach the pubusb service, empty to disable pubsub functionality: <""|internal|x.y.z.y:1234>
+	"users": "",							// address where to reach the user service, empty to disable user profile functionality: <""|internal|x.y.z.y:1234>
+	"aliases": "",							// address where to reach the aliases service, empty to disable aliases functionality: <""|internal|x.y.z.y:1234>
+	"cdrstats": "",							// address where to reach the cdrstats service, empty to disable stats functionality<""|internal|x.y.z.y:1234>
+	"cdr_replication":[],					// replicate the raw CDR to a number of servers
+},
+
+
+"cdrstats": {
+	"enabled": false,						// starts the cdrstats service: <true|false>
+	"save_interval": "1m",					// interval to save changed stats into dataDb storage
+},
+
+
+"cdre": {
+	"*default": {
+		"cdr_format": "csv",							// exported CDRs format <csv>
+		"field_separator": ",",
+		"data_usage_multiply_factor": 1,				// multiply data usage before export (eg: convert from KBytes to Bytes)
+		"sms_usage_multiply_factor": 1,					// multiply data usage before export (eg: convert from SMS unit to call duration in some billing systems)
+		"generic_usage_multiply_factor": 1,					// multiply data usage before export (eg: convert from GENERIC unit to call duration in some billing systems)
+		"cost_multiply_factor": 1,						// multiply cost before export, eg: add VAT
+		"cost_rounding_decimals": -1,					// rounding decimals for Cost values. -1 to disable rounding
+		"cost_shift_digits": 0,							// shift digits in the cost on export (eg: convert from EUR to cents)
+		"mask_destination_id": "MASKED_DESTINATIONS",	// destination id containing called addresses to be masked on export
+		"mask_length": 0,								// length of the destination suffix to be masked
+		"export_dir": "/var/log/cgrates/cdre",			// path where the exported CDRs will be placed
+		"header_fields": [],							// template of the exported header fields
+		"content_fields": [								// template of the exported content fields
+			{"tag": "CgrId", "cdr_field_id": "cgrid", "type": "cdrfield", "value": "cgrid"},
+			{"tag":"RunId", "cdr_field_id": "mediation_runid", "type": "cdrfield", "value": "mediation_runid"},
+			{"tag":"Tor", "cdr_field_id": "tor", "type": "cdrfield", "value": "tor"},
+			{"tag":"AccId", "cdr_field_id": "accid", "type": "cdrfield", "value": "accid"},
+			{"tag":"ReqType", "cdr_field_id": "reqtype", "type": "cdrfield", "value": "reqtype"},
+			{"tag":"Direction", "cdr_field_id": "direction", "type": "cdrfield", "value": "direction"},
+			{"tag":"Tenant", "cdr_field_id": "tenant", "type": "cdrfield", "value": "tenant"},
+			{"tag":"Category", "cdr_field_id": "category", "type": "cdrfield", "value": "category"},
+			{"tag":"Account", "cdr_field_id": "account", "type": "cdrfield", "value": "account"},
+			{"tag":"Subject", "cdr_field_id": "subject", "type": "cdrfield", "value": "subject"},
+			{"tag":"Destination", "cdr_field_id": "destination", "type": "cdrfield", "value": "destination"},
+			{"tag":"SetupTime", "cdr_field_id": "setup_time", "type": "cdrfield", "value": "setup_time", "layout": "2006-01-02T15:04:05Z07:00"},
+			{"tag":"AnswerTime", "cdr_field_id": "answer_time", "type": "cdrfield", "value": "answer_time", "layout": "2006-01-02T15:04:05Z07:00"},
+			{"tag":"Usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "usage"},
+			{"tag":"Cost", "cdr_field_id": "cost", "type": "cdrfield", "value": "cost"},
+		],
+		"trailer_fields": [],							// template of the exported trailer fields
+	}
+},
+
+
+"cdrc": {
+	"*default": {
+		"enabled": false,							// enable CDR client functionality
+		"dry_run": false,							// do not send the CDRs to CDRS, just parse them
+		"cdrs": "internal",							// address where to reach CDR server. <internal|x.y.z.y:1234>
+		"cdr_format": "csv",						// CDR file format <csv|freeswitch_csv|fwv|opensips_flatstore>
+		"field_separator": ",",						// separator used in case of csv files
+		"timezone": "",								// timezone for timestamps where not specified <""|UTC|Local|$IANA_TZ_DB>
+		"run_delay": 0,								// sleep interval in seconds between consecutive runs, 0 to use automation via inotify
+		"max_open_files": 1024,						// maximum simultaneous files to process, 0 for unlimited
+		"data_usage_multiply_factor": 1024,			// conversion factor for data usage
+		"cdr_in_dir": "/var/log/cgrates/cdrc/in",	// absolute path towards the directory where the CDRs are stored
+		"cdr_out_dir": "/var/log/cgrates/cdrc/out",	// absolute path towards the directory where processed CDRs will be moved
+		"failed_calls_prefix": "missed_calls",		// used in case of flatstore CDRs to avoid searching for BYE records
+		"cdr_source_id": "freeswitch_csv",			// free form field, tag identifying the source of the CDRs within CDRS database
+		"cdr_filter": "",							// filter CDR records to import
+		"partial_record_cache": "10s",				// duration to cache partial records when not pairing
+		"header_fields": [],						// template of the import header fields
+		"content_fields":[							// import content_fields template, tag will match internally CDR field, in case of .csv value will be represented by index of the field value
+			{"tag": "tor", "cdr_field_id": "tor", "type": "cdrfield", "value": "2", "mandatory": true},
+			{"tag": "accid", "cdr_field_id": "accid", "type": "cdrfield", "value": "3", "mandatory": true},
+			{"tag": "reqtype", "cdr_field_id": "reqtype", "type": "cdrfield", "value": "4", "mandatory": true},
+			{"tag": "direction", "cdr_field_id": "direction", "type": "cdrfield", "value": "5", "mandatory": true},
+			{"tag": "tenant", "cdr_field_id": "tenant", "type": "cdrfield", "value": "6", "mandatory": true},
+			{"tag": "category", "cdr_field_id": "category", "type": "cdrfield", "value": "7", "mandatory": true},
+			{"tag": "account", "cdr_field_id": "account", "type": "cdrfield", "value": "8", "mandatory": true},
+			{"tag": "subject", "cdr_field_id": "subject", "type": "cdrfield", "value": "9", "mandatory": true},
+			{"tag": "destination", "cdr_field_id": "destination", "type": "cdrfield", "value": "10", "mandatory": true},
+			{"tag": "setup_time", "cdr_field_id": "setup_time", "type": "cdrfield", "value": "11", "mandatory": true},
+			{"tag": "answer_time", "cdr_field_id": "answer_time", "type": "cdrfield", "value": "12", "mandatory": true},
+			{"tag": "usage", "cdr_field_id": "usage", "type": "cdrfield", "value": "13", "mandatory": true},
+		],
+		"trailer_fields": [],							// template of the import trailer fields
+	}
+},
+
+
+"sm_freeswitch": {
+	"enabled": false,				// starts SessionManager service: <true|false>
+	"rater": "internal",			// address where to reach the Rater <""|internal|127.0.0.1:2013>
+	"cdrs": "internal",				// address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234>
+	"create_cdr": false,			// create CDR out of events and sends them to CDRS component
+	"extra_fields": [],				// extra fields to store in auth/CDRs when creating them
+	"debit_interval": "10s",		// interval to perform debits on.
+	"min_call_duration": "0s",		// only authorize calls with allowed duration higher than this
+	"max_call_duration": "3h",		// maximum call duration a prepaid call can last
+	"min_dur_low_balance": "5s",	// threshold which will trigger low balance warnings for prepaid calls (needs to be lower than debit_interval)
+	"low_balance_ann_file": "",		// file to be played when low balance is reached for prepaid calls
+	"empty_balance_context": "",	// if defined, prepaid calls will be transfered to this context on empty balance
+	"empty_balance_ann_file": "",	// file to be played before disconnecting prepaid calls on empty balance (applies only if no context defined)
+	"subscribe_park": true,			// subscribe via fsock to receive park events
+	"channel_sync_interval": "5m",	// sync channels with freeswitch regularly
+	"connections":[					// instantiate connections to multiple FreeSWITCH servers
+		{"server": "127.0.0.1:8021", "password": "ClueCon", "reconnects": 5}
+	],
+},
+
+
+"sm_kamailio": {
+	"enabled": false,				// starts SessionManager service: <true|false>
+	"rater": "internal",			// address where to reach the Rater <""|internal|127.0.0.1:2013>
+	"cdrs": "internal",				// address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234>
+	"create_cdr": false,			// create CDR out of events and sends them to CDRS component
+	"debit_interval": "10s",		// interval to perform debits on.
+	"min_call_duration": "0s",		// only authorize calls with allowed duration higher than this
+	"max_call_duration": "3h",		// maximum call duration a prepaid call can last
+	"connections":[					// instantiate connections to multiple Kamailio servers
+		{"evapi_addr": "127.0.0.1:8448", "reconnects": 5}
+	],
+},
+
+
+"sm_opensips": {
+	"enabled": false,					// starts SessionManager service: <true|false>
+	"listen_udp": "127.0.0.1:2020",		// address where to listen for datagram events coming from OpenSIPS
+	"rater": "internal",				// address where to reach the Rater <""|internal|127.0.0.1:2013>
+	"cdrs": "internal",					// address where to reach CDR Server, empty to disable CDR capturing <""|internal|x.y.z.y:1234>
+	"reconnects": 5,					// number of reconnects if connection is lost
+	"create_cdr": false,				// create CDR out of events and sends them to CDRS component
+	"debit_interval": "10s",			// interval to perform debits on.
+	"min_call_duration": "0s",			// only authorize calls with allowed duration higher than this
+	"max_call_duration": "3h",			// maximum call duration a prepaid call can last
+	"events_subscribe_interval": "60s",	// automatic events subscription to OpenSIPS, 0 to disable it
+	"mi_addr": "127.0.0.1:8020",		// address where to reach OpenSIPS MI to send session disconnects
+},
+
+
+"historys": {
+	"enabled": false,							// starts History service: <true|false>.
+	"history_dir": "/var/log/cgrates/history",	// location on disk where to store history files.
+	"save_interval": "1s",						// interval to save changed cache into .git archive
+},
+
+
+"pubsubs": {
+	"enabled": false,							// starts PubSub service: <true|false>.
+},
+
+
+"aliases": {
+	"enabled": false,							// starts Aliases service: <true|false>.
+},
+
+
+"users": {
+	"enabled": false,							// starts User service: <true|false>.
+	"indexes": [],								// user profile field indexes
+},
+
+
+"mailer": {
+	"server": "localhost",								// the server to use when sending emails out
+	"auth_user": "cgrates",								// authenticate to email server using this user
+	"auth_passwd": "CGRateS.org",						// authenticate to email server with this password
+	"from_address": "cgr-mailer@localhost.localdomain"	// from address used when sending emails out
+},
+
+
+}`

config/config_json.go

@@ -0,0 +1,294 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+
+	"github.com/DisposaBoy/JsonConfigReader"
+)
+
+const (
+	GENERAL_JSN     = "general"
+	LISTEN_JSN      = "listen"
+	TPDB_JSN        = "tariffplan_db"
+	DATADB_JSN      = "data_db"
+	STORDB_JSN      = "stor_db"
+	BALANCER_JSN    = "balancer"
+	RATER_JSN       = "rater"
+	SCHEDULER_JSN   = "scheduler"
+	CDRS_JSN        = "cdrs"
+	MEDIATOR_JSN    = "mediator"
+	CDRSTATS_JSN    = "cdrstats"
+	CDRE_JSN        = "cdre"
+	CDRC_JSN        = "cdrc"
+	SMFS_JSN        = "sm_freeswitch"
+	SMKAM_JSN       = "sm_kamailio"
+	SMOSIPS_JSN     = "sm_opensips"
+	SM_JSN          = "session_manager"
+	FS_JSN          = "freeswitch"
+	KAMAILIO_JSN    = "kamailio"
+	OSIPS_JSN       = "opensips"
+	HISTSERV_JSN    = "historys"
+	PUBSUBSERV_JSN  = "pubsubs"
+	ALIASESSERV_JSN = "aliases"
+	USERSERV_JSN    = "users"
+	MAILER_JSN      = "mailer"
+)
+
+// Loads the json config out of io.Reader, eg other sources than file, maybe over http
+func NewCgrJsonCfgFromReader(r io.Reader) (*CgrJsonCfg, error) {
+	var cgrJsonCfg CgrJsonCfg
+	jr := JsonConfigReader.New(r)
+	if err := json.NewDecoder(jr).Decode(&cgrJsonCfg); err != nil {
+		return nil, err
+	}
+	return &cgrJsonCfg, nil
+}
+
+// Loads the config out of file
+func NewCgrJsonCfgFromFile(fpath string) (*CgrJsonCfg, error) {
+	cfgFile, err := os.Open(fpath)
+	if err != nil {
+		return nil, err
+	}
+	defer cfgFile.Close()
+	return NewCgrJsonCfgFromReader(cfgFile)
+}
+
+// Main object holding the loaded config as section raw messages
+type CgrJsonCfg map[string]*json.RawMessage
+
+func (self CgrJsonCfg) GeneralJsonCfg() (*GeneralJsonCfg, error) {
+	rawCfg, hasKey := self[GENERAL_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(GeneralJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) ListenJsonCfg() (*ListenJsonCfg, error) {
+	rawCfg, hasKey := self[LISTEN_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(ListenJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) DbJsonCfg(section string) (*DbJsonCfg, error) {
+	rawCfg, hasKey := self[section]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(DbJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) BalancerJsonCfg() (*BalancerJsonCfg, error) {
+	rawCfg, hasKey := self[BALANCER_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(BalancerJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) RaterJsonCfg() (*RaterJsonCfg, error) {
+	rawCfg, hasKey := self[RATER_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(RaterJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) SchedulerJsonCfg() (*SchedulerJsonCfg, error) {
+	rawCfg, hasKey := self[SCHEDULER_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(SchedulerJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) CdrsJsonCfg() (*CdrsJsonCfg, error) {
+	rawCfg, hasKey := self[CDRS_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(CdrsJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) CdrStatsJsonCfg() (*CdrStatsJsonCfg, error) {
+	rawCfg, hasKey := self[CDRSTATS_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(CdrStatsJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) CdreJsonCfgs() (map[string]*CdreJsonCfg, error) {
+	rawCfg, hasKey := self[CDRE_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := make(map[string]*CdreJsonCfg)
+	if err := json.Unmarshal(*rawCfg, &cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) CdrcJsonCfg() (map[string]*CdrcJsonCfg, error) {
+	rawCfg, hasKey := self[CDRC_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := make(map[string]*CdrcJsonCfg)
+	if err := json.Unmarshal(*rawCfg, &cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) SmFsJsonCfg() (*SmFsJsonCfg, error) {
+	rawCfg, hasKey := self[SMFS_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(SmFsJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) SmKamJsonCfg() (*SmKamJsonCfg, error) {
+	rawCfg, hasKey := self[SMKAM_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(SmKamJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) SmOsipsJsonCfg() (*SmOsipsJsonCfg, error) {
+	rawCfg, hasKey := self[SMOSIPS_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(SmOsipsJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) HistServJsonCfg() (*HistServJsonCfg, error) {
+	rawCfg, hasKey := self[HISTSERV_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(HistServJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) PubSubServJsonCfg() (*PubSubServJsonCfg, error) {
+	rawCfg, hasKey := self[PUBSUBSERV_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(PubSubServJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) AliasesServJsonCfg() (*AliasesServJsonCfg, error) {
+	rawCfg, hasKey := self[ALIASESSERV_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(AliasesServJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) UserServJsonCfg() (*UserServJsonCfg, error) {
+	rawCfg, hasKey := self[USERSERV_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(UserServJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
+
+func (self CgrJsonCfg) MailerJsonCfg() (*MailerJsonCfg, error) {
+	rawCfg, hasKey := self[MAILER_JSN]
+	if !hasKey {
+		return nil, nil
+	}
+	cfg := new(MailerJsonCfg)
+	if err := json.Unmarshal(*rawCfg, cfg); err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}

config/config_json_test.go

@@ -0,0 +1,518 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+var dfCgrJsonCfg *CgrJsonCfg
+
+// Loads up the default configuration and  tests it's sections one by one
+func TestDfNewdfCgrJsonCfgFromReader(t *testing.T) {
+	var err error
+	if dfCgrJsonCfg, err = NewCgrJsonCfgFromReader(strings.NewReader(CGRATES_CFG_JSON)); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestDfGeneralJsonCfg(t *testing.T) {
+	eCfg := &GeneralJsonCfg{
+		Http_skip_tls_verify: utils.BoolPointer(false),
+		Rounding_decimals:    utils.IntPointer(10),
+		Dbdata_encoding:      utils.StringPointer("msgpack"),
+		Tpexport_dir:         utils.StringPointer("/var/log/cgrates/tpe"),
+		Default_reqtype:      utils.StringPointer(utils.META_RATED),
+		Default_category:     utils.StringPointer("call"),
+		Default_tenant:       utils.StringPointer("cgrates.org"),
+		Default_subject:      utils.StringPointer("cgrates"),
+		Default_timezone:     utils.StringPointer("Local"),
+		Connect_attempts:     utils.IntPointer(3),
+		Reconnects:           utils.IntPointer(-1),
+		Response_cache_ttl:   utils.StringPointer("3s"),
+		Internal_ttl:         utils.StringPointer("2m")}
+	if gCfg, err := dfCgrJsonCfg.GeneralJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, gCfg) {
+		t.Error("Received: ", gCfg)
+	}
+}
+
+func TestDfListenJsonCfg(t *testing.T) {
+	eCfg := &ListenJsonCfg{
+		Rpc_json: utils.StringPointer("127.0.0.1:2012"),
+		Rpc_gob:  utils.StringPointer("127.0.0.1:2013"),
+		Http:     utils.StringPointer("127.0.0.1:2080")}
+	if cfg, err := dfCgrJsonCfg.ListenJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfDbJsonCfg(t *testing.T) {
+	eCfg := &DbJsonCfg{
+		Db_type:   utils.StringPointer("redis"),
+		Db_host:   utils.StringPointer("127.0.0.1"),
+		Db_port:   utils.IntPointer(6379),
+		Db_name:   utils.StringPointer("10"),
+		Db_user:   utils.StringPointer(""),
+		Db_passwd: utils.StringPointer(""),
+	}
+	if cfg, err := dfCgrJsonCfg.DbJsonCfg(TPDB_JSN); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+	eCfg = &DbJsonCfg{
+		Db_type:           utils.StringPointer("redis"),
+		Db_host:           utils.StringPointer("127.0.0.1"),
+		Db_port:           utils.IntPointer(6379),
+		Db_name:           utils.StringPointer("11"),
+		Db_user:           utils.StringPointer(""),
+		Db_passwd:         utils.StringPointer(""),
+		Load_history_size: utils.IntPointer(10),
+	}
+	if cfg, err := dfCgrJsonCfg.DbJsonCfg(DATADB_JSN); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+	eCfg = &DbJsonCfg{
+		Db_type:        utils.StringPointer("mysql"),
+		Db_host:        utils.StringPointer("127.0.0.1"),
+		Db_port:        utils.IntPointer(3306),
+		Db_name:        utils.StringPointer("cgrates"),
+		Db_user:        utils.StringPointer("cgrates"),
+		Db_passwd:      utils.StringPointer("CGRateS.org"),
+		Max_open_conns: utils.IntPointer(100),
+		Max_idle_conns: utils.IntPointer(10),
+	}
+	if cfg, err := dfCgrJsonCfg.DbJsonCfg(STORDB_JSN); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfBalancerJsonCfg(t *testing.T) {
+	eCfg := &BalancerJsonCfg{Enabled: utils.BoolPointer(false)}
+	if cfg, err := dfCgrJsonCfg.BalancerJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfRaterJsonCfg(t *testing.T) {
+	eCfg := &RaterJsonCfg{Enabled: utils.BoolPointer(false), Balancer: utils.StringPointer(""), Cdrstats: utils.StringPointer(""),
+		Historys: utils.StringPointer(""), Pubsubs: utils.StringPointer(""), Users: utils.StringPointer(""), Aliases: utils.StringPointer("")}
+	if cfg, err := dfCgrJsonCfg.RaterJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Errorf("Received: %+v", cfg)
+	}
+}
+
+func TestDfSchedulerJsonCfg(t *testing.T) {
+	eCfg := &SchedulerJsonCfg{Enabled: utils.BoolPointer(false)}
+	if cfg, err := dfCgrJsonCfg.SchedulerJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfCdrsJsonCfg(t *testing.T) {
+	eCfg := &CdrsJsonCfg{
+		Enabled:         utils.BoolPointer(false),
+		Extra_fields:    utils.StringSlicePointer([]string{}),
+		Store_cdrs:      utils.BoolPointer(true),
+		Rater:           utils.StringPointer("internal"),
+		Pubsubs:         utils.StringPointer(""),
+		Users:           utils.StringPointer(""),
+		Aliases:         utils.StringPointer(""),
+		Cdrstats:        utils.StringPointer(""),
+		Cdr_replication: &[]*CdrReplicationJsonCfg{},
+	}
+	if cfg, err := dfCgrJsonCfg.CdrsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Errorf("Received: %+v", *cfg)
+	}
+}
+
+func TestDfCdrStatsJsonCfg(t *testing.T) {
+	eCfg := &CdrStatsJsonCfg{
+		Enabled:       utils.BoolPointer(false),
+		Save_Interval: utils.StringPointer("1m"),
+	}
+	if cfg, err := dfCgrJsonCfg.CdrStatsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", *cfg)
+	}
+}
+
+func TestDfCdreJsonCfgs(t *testing.T) {
+	eFields := []*CdrFieldJsonCfg{}
+	eContentFlds := []*CdrFieldJsonCfg{
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("CgrId"),
+			Cdr_field_id: utils.StringPointer("cgrid"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("cgrid")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("RunId"),
+			Cdr_field_id: utils.StringPointer("mediation_runid"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("mediation_runid")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Tor"),
+			Cdr_field_id: utils.StringPointer("tor"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("tor")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("AccId"),
+			Cdr_field_id: utils.StringPointer("accid"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("accid")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("ReqType"),
+			Cdr_field_id: utils.StringPointer("reqtype"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("reqtype")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Direction"),
+			Cdr_field_id: utils.StringPointer("direction"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("direction")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Tenant"),
+			Cdr_field_id: utils.StringPointer("tenant"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("tenant")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Category"),
+			Cdr_field_id: utils.StringPointer("category"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("category")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Account"),
+			Cdr_field_id: utils.StringPointer("account"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("account")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Subject"),
+			Cdr_field_id: utils.StringPointer("subject"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("subject")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Destination"),
+			Cdr_field_id: utils.StringPointer("destination"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("destination")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("SetupTime"),
+			Cdr_field_id: utils.StringPointer("setup_time"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("setup_time"),
+			Layout:       utils.StringPointer("2006-01-02T15:04:05Z07:00")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("AnswerTime"),
+			Cdr_field_id: utils.StringPointer("answer_time"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("answer_time"),
+			Layout:       utils.StringPointer("2006-01-02T15:04:05Z07:00")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Usage"),
+			Cdr_field_id: utils.StringPointer("usage"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("usage")},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("Cost"),
+			Cdr_field_id: utils.StringPointer("cost"),
+			Type:         utils.StringPointer("cdrfield"),
+			Value:        utils.StringPointer("cost")},
+	}
+	eCfg := map[string]*CdreJsonCfg{
+		utils.META_DEFAULT: &CdreJsonCfg{
+			Cdr_format:                    utils.StringPointer("csv"),
+			Field_separator:               utils.StringPointer(","),
+			Data_usage_multiply_factor:    utils.Float64Pointer(1.0),
+			Sms_usage_multiply_factor:     utils.Float64Pointer(1.0),
+			Generic_usage_multiply_factor: utils.Float64Pointer(1.0),
+			Cost_multiply_factor:          utils.Float64Pointer(1.0),
+			Cost_rounding_decimals:        utils.IntPointer(-1),
+			Cost_shift_digits:             utils.IntPointer(0),
+			Mask_destination_id:           utils.StringPointer("MASKED_DESTINATIONS"),
+			Mask_length:                   utils.IntPointer(0),
+			Export_dir:                    utils.StringPointer("/var/log/cgrates/cdre"),
+			Header_fields:                 &eFields,
+			Content_fields:                &eContentFlds,
+			Trailer_fields:                &eFields,
+		},
+	}
+	if cfg, err := dfCgrJsonCfg.CdreJsonCfgs(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfCdrcJsonCfg(t *testing.T) {
+	eFields := []*CdrFieldJsonCfg{}
+	cdrFields := []*CdrFieldJsonCfg{
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("tor"), Cdr_field_id: utils.StringPointer("tor"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("2"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("accid"), Cdr_field_id: utils.StringPointer("accid"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("3"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("reqtype"), Cdr_field_id: utils.StringPointer("reqtype"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("4"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("direction"), Cdr_field_id: utils.StringPointer("direction"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("5"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("tenant"), Cdr_field_id: utils.StringPointer("tenant"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("6"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("category"), Cdr_field_id: utils.StringPointer("category"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("7"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("account"), Cdr_field_id: utils.StringPointer("account"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("8"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("subject"), Cdr_field_id: utils.StringPointer("subject"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("9"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("destination"), Cdr_field_id: utils.StringPointer("destination"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("10"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("setup_time"), Cdr_field_id: utils.StringPointer("setup_time"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("11"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("answer_time"), Cdr_field_id: utils.StringPointer("answer_time"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("12"), Mandatory: utils.BoolPointer(true)},
+		&CdrFieldJsonCfg{Tag: utils.StringPointer("usage"), Cdr_field_id: utils.StringPointer("usage"), Type: utils.StringPointer(utils.CDRFIELD),
+			Value: utils.StringPointer("13"), Mandatory: utils.BoolPointer(true)},
+	}
+	eCfg := map[string]*CdrcJsonCfg{
+		"*default": &CdrcJsonCfg{
+			Enabled:                    utils.BoolPointer(false),
+			Dry_run:                    utils.BoolPointer(false),
+			Cdrs:                       utils.StringPointer("internal"),
+			Cdr_format:                 utils.StringPointer("csv"),
+			Field_separator:            utils.StringPointer(","),
+			Timezone:                   utils.StringPointer(""),
+			Run_delay:                  utils.IntPointer(0),
+			Max_open_files:             utils.IntPointer(1024),
+			Data_usage_multiply_factor: utils.Float64Pointer(1024.0),
+			Cdr_in_dir:                 utils.StringPointer("/var/log/cgrates/cdrc/in"),
+			Cdr_out_dir:                utils.StringPointer("/var/log/cgrates/cdrc/out"),
+			Failed_calls_prefix:        utils.StringPointer("missed_calls"),
+			Cdr_source_id:              utils.StringPointer("freeswitch_csv"),
+			Cdr_filter:                 utils.StringPointer(""),
+			Partial_record_cache:       utils.StringPointer("10s"),
+			Header_fields:              &eFields,
+			Content_fields:             &cdrFields,
+			Trailer_fields:             &eFields,
+		},
+	}
+	if cfg, err := dfCgrJsonCfg.CdrcJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg["*default"])
+	}
+}
+
+func TestSmFsJsonCfg(t *testing.T) {
+	eCfg := &SmFsJsonCfg{
+		Enabled:                utils.BoolPointer(false),
+		Rater:                  utils.StringPointer("internal"),
+		Cdrs:                   utils.StringPointer("internal"),
+		Create_cdr:             utils.BoolPointer(false),
+		Extra_fields:           utils.StringSlicePointer([]string{}),
+		Debit_interval:         utils.StringPointer("10s"),
+		Min_call_duration:      utils.StringPointer("0s"),
+		Max_call_duration:      utils.StringPointer("3h"),
+		Min_dur_low_balance:    utils.StringPointer("5s"),
+		Low_balance_ann_file:   utils.StringPointer(""),
+		Empty_balance_context:  utils.StringPointer(""),
+		Empty_balance_ann_file: utils.StringPointer(""),
+		Subscribe_park:         utils.BoolPointer(true),
+		Channel_sync_interval:  utils.StringPointer("5m"),
+		Connections: &[]*FsConnJsonCfg{
+			&FsConnJsonCfg{
+				Server:     utils.StringPointer("127.0.0.1:8021"),
+				Password:   utils.StringPointer("ClueCon"),
+				Reconnects: utils.IntPointer(5),
+			}},
+	}
+	if cfg, err := dfCgrJsonCfg.SmFsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestSmKamJsonCfg(t *testing.T) {
+	eCfg := &SmKamJsonCfg{
+		Enabled:           utils.BoolPointer(false),
+		Rater:             utils.StringPointer("internal"),
+		Cdrs:              utils.StringPointer("internal"),
+		Create_cdr:        utils.BoolPointer(false),
+		Debit_interval:    utils.StringPointer("10s"),
+		Min_call_duration: utils.StringPointer("0s"),
+		Max_call_duration: utils.StringPointer("3h"),
+		Connections: &[]*KamConnJsonCfg{
+			&KamConnJsonCfg{
+				Evapi_addr: utils.StringPointer("127.0.0.1:8448"),
+				Reconnects: utils.IntPointer(5),
+			},
+		},
+	}
+	if cfg, err := dfCgrJsonCfg.SmKamJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestSmOsipsJsonCfg(t *testing.T) {
+	eCfg := &SmOsipsJsonCfg{
+		Enabled:                   utils.BoolPointer(false),
+		Listen_udp:                utils.StringPointer("127.0.0.1:2020"),
+		Rater:                     utils.StringPointer("internal"),
+		Cdrs:                      utils.StringPointer("internal"),
+		Create_cdr:                utils.BoolPointer(false),
+		Debit_interval:            utils.StringPointer("10s"),
+		Min_call_duration:         utils.StringPointer("0s"),
+		Max_call_duration:         utils.StringPointer("3h"),
+		Events_subscribe_interval: utils.StringPointer("60s"),
+		Mi_addr:                   utils.StringPointer("127.0.0.1:8020"),
+	}
+	if cfg, err := dfCgrJsonCfg.SmOsipsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfHistServJsonCfg(t *testing.T) {
+	eCfg := &HistServJsonCfg{
+		Enabled:       utils.BoolPointer(false),
+		History_dir:   utils.StringPointer("/var/log/cgrates/history"),
+		Save_interval: utils.StringPointer("1s"),
+	}
+	if cfg, err := dfCgrJsonCfg.HistServJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfPubSubServJsonCfg(t *testing.T) {
+	eCfg := &PubSubServJsonCfg{
+		Enabled: utils.BoolPointer(false),
+	}
+	if cfg, err := dfCgrJsonCfg.PubSubServJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfAliasesServJsonCfg(t *testing.T) {
+	eCfg := &AliasesServJsonCfg{
+		Enabled: utils.BoolPointer(false),
+	}
+	if cfg, err := dfCgrJsonCfg.AliasesServJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfUserServJsonCfg(t *testing.T) {
+	eCfg := &UserServJsonCfg{
+		Enabled: utils.BoolPointer(false),
+		Indexes: utils.StringSlicePointer([]string{}),
+	}
+	if cfg, err := dfCgrJsonCfg.UserServJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestDfMailerJsonCfg(t *testing.T) {
+
+	eCfg := &MailerJsonCfg{
+		Server:       utils.StringPointer("localhost"),
+		Auth_user:    utils.StringPointer("cgrates"),
+		Auth_passwd:  utils.StringPointer("CGRateS.org"),
+		From_address: utils.StringPointer("cgr-mailer@localhost.localdomain"),
+	}
+	if cfg, err := dfCgrJsonCfg.MailerJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, cfg) {
+		t.Error("Received: ", cfg)
+	}
+}
+
+func TestNewCgrJsonCfgFromFile(t *testing.T) {
+	cgrJsonCfg, err := NewCgrJsonCfgFromFile("cfg_data.json")
+	if err != nil {
+		t.Error(err)
+	}
+	eCfg := &GeneralJsonCfg{Default_reqtype: utils.StringPointer(utils.META_PSEUDOPREPAID)}
+	if gCfg, err := cgrJsonCfg.GeneralJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfg, gCfg) {
+		t.Error("Received: ", gCfg)
+	}
+	cdrFields := []*CdrFieldJsonCfg{
+		&CdrFieldJsonCfg{Cdr_field_id: utils.StringPointer("tor"), Value: utils.StringPointer("~7:s/^(voice|data|sms|generic)$/*$1/")},
+		&CdrFieldJsonCfg{Cdr_field_id: utils.StringPointer("answer_time"), Value: utils.StringPointer("1")},
+		&CdrFieldJsonCfg{Cdr_field_id: utils.StringPointer("usage"), Value: utils.StringPointer(`~9:s/^(\d+)$/${1}s/`)},
+	}
+	eCfgCdrc := map[string]*CdrcJsonCfg{
+		"CDRC-CSV1": &CdrcJsonCfg{
+			Enabled:       utils.BoolPointer(true),
+			Cdr_in_dir:    utils.StringPointer("/tmp/cgrates/cdrc1/in"),
+			Cdr_out_dir:   utils.StringPointer("/tmp/cgrates/cdrc1/out"),
+			Cdr_source_id: utils.StringPointer("csv1"),
+		},
+		"CDRC-CSV2": &CdrcJsonCfg{
+			Enabled:                    utils.BoolPointer(true),
+			Data_usage_multiply_factor: utils.Float64Pointer(0.000976563),
+			Run_delay:                  utils.IntPointer(1),
+			Cdr_in_dir:                 utils.StringPointer("/tmp/cgrates/cdrc2/in"),
+			Cdr_out_dir:                utils.StringPointer("/tmp/cgrates/cdrc2/out"),
+			Cdr_source_id:              utils.StringPointer("csv2"),
+			Content_fields:             &cdrFields,
+		},
+	}
+	if cfg, err := cgrJsonCfg.CdrcJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfgCdrc, cfg) {
+		t.Error("Received: ", cfg["CDRC-CSV2"])
+	}
+	eCfgSmFs := &SmFsJsonCfg{
+		Enabled: utils.BoolPointer(true),
+		Connections: &[]*FsConnJsonCfg{
+			&FsConnJsonCfg{
+				Server:     utils.StringPointer("1.2.3.4:8021"),
+				Password:   utils.StringPointer("ClueCon"),
+				Reconnects: utils.IntPointer(5),
+			},
+			&FsConnJsonCfg{
+				Server:     utils.StringPointer("2.3.4.5:8021"),
+				Password:   utils.StringPointer("ClueCon"),
+				Reconnects: utils.IntPointer(5),
+			},
+		},
+	}
+	if smFsCfg, err := cgrJsonCfg.SmFsJsonCfg(); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCfgSmFs, smFsCfg) {
+		t.Error("Received: ", smFsCfg)
+	}
+}

config/config_test.go

@@ -1,6 +1,6 @@
 /*
-Rating system designed to be used in VoIP Carriers World
-Copyright (C) 2013 ITsysCOM
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -19,117 +19,42 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>
 package config
 
 import (
-	"fmt"
+	"reflect"
 	"testing"
 )
 
-func TestConfig(t *testing.T) {
-	cfgPth := "test_data.txt"
-	cfg, err := NewCGRConfig(&cfgPth)
-	if err != nil {
-		t.Log(fmt.Sprintf("Could not parse config: %s!", err))
-		t.FailNow()
+var cfg *CGRConfig
+
+func TestConfigSharing(t *testing.T) {
+	cfg, _ = NewDefaultCGRConfig()
+	SetCgrConfig(cfg)
+	cfgReturn := CgrConfig()
+	if !reflect.DeepEqual(cfgReturn, cfg) {
+		t.Errorf("Retrieved %v, Expected %v", cfgReturn, cfg)
 	}
-	if cfg.DataDBType != "test" ||
-		cfg.DataDBHost != "test" ||
-		cfg.DataDBPort != "test" ||
-		cfg.DataDBName != "test" ||
-		cfg.DataDBUser != "test" ||
-		cfg.DataDBPass != "test" ||
-		cfg.LogDBType != "test" ||
-		cfg.LogDBHost != "test" ||
-		cfg.LogDBPort != "test" ||
-		cfg.LogDBName != "test" ||
-		cfg.LogDBUser != "test" ||
-		cfg.LogDBPass != "test" ||
-		cfg.RaterEnabled != true ||
-		cfg.RaterBalancer != "test" ||
-		cfg.RaterListen != "test" ||
-		cfg.RaterRPCEncoding != "test" ||
-		cfg.BalancerEnabled != true ||
-		cfg.BalancerListen != "test" ||
-		cfg.BalancerRPCEncoding != "test" ||
-		cfg.SchedulerEnabled != true ||
-		cfg.SMEnabled != true ||
-		cfg.SMSwitchType != "test" ||
-		cfg.SMRater != "test" ||
-		cfg.SMDebitInterval != 11 ||
-		cfg.SMRPCEncoding != "test" ||
-		cfg.MediatorEnabled != true ||
-		cfg.MediatorCDRInDir != "test" ||
-		cfg.MediatorCDROutDir != "test" ||
-		cfg.MediatorRater != "test" ||
-		cfg.MediatorRPCEncoding != "test" ||
-		cfg.MediatorSkipDB != true ||
-		cfg.MediatorPseudoprepaid != true ||
-		cfg.FreeswitchServer != "test" ||
-		cfg.FreeswitchPass != "test" ||
-		cfg.FreeswitchDirectionIdx != "test" ||
-		cfg.FreeswitchTORIdx != "test" ||
-		cfg.FreeswitchTenantIdx != "test" ||
-		cfg.FreeswitchSubjectIdx != "test" ||
-		cfg.FreeswitchAccountIdx != "test" ||
-		cfg.FreeswitchDestIdx != "test" ||
-		cfg.FreeswitchTimeStartIdx != "test" ||
-		cfg.FreeswitchDurationIdx != "test" ||
-		cfg.FreeswitchUUIDIdx != "test" {
-			t.Log(cfg.DataDBType)
-			t.Log(cfg.DataDBHost)
-			t.Log(cfg.DataDBPort)
-			t.Log(cfg.DataDBName)
-			t.Log(cfg.DataDBUser)
-			t.Log(cfg.DataDBPass)
-			t.Log(cfg.LogDBType)
-			t.Log(cfg.LogDBHost)
-			t.Log(cfg.LogDBPort)
-			t.Log(cfg.LogDBName)
-			t.Log(cfg.LogDBUser)
-			t.Log(cfg.LogDBPass)
-			t.Log(cfg.RaterEnabled)
-			t.Log(cfg.RaterBalancer)
-			t.Log(cfg.RaterListen) 
-			t.Log(cfg.RaterRPCEncoding)
-			t.Log(cfg.BalancerEnabled)
-			t.Log(cfg.BalancerListen) 
-			t.Log(cfg.BalancerRPCEncoding)
-			t.Log(cfg.SchedulerEnabled)
-			t.Log(cfg.SMEnabled)
-			t.Log(cfg.SMSwitchType) 
-			t.Log(cfg.SMRater)
-			t.Log(cfg.SMDebitInterval)
-			t.Log(cfg.SMRPCEncoding)
-			t.Log(cfg.MediatorEnabled)
-			t.Log(cfg.MediatorCDRInDir)
-			t.Log(cfg.MediatorCDROutDir) 
-			t.Log(cfg.MediatorRater)
-			t.Log(cfg.MediatorRPCEncoding)
-			t.Log(cfg.MediatorSkipDB) 
-			t.Log(cfg.MediatorPseudoprepaid) 
-			t.Log(cfg.FreeswitchServer) 
-			t.Log(cfg.FreeswitchPass)
-			t.Log(cfg.FreeswitchDirectionIdx)
-			t.Log(cfg.FreeswitchTORIdx) 
-			t.Log(cfg.FreeswitchTenantIdx) 
-			t.Log(cfg.FreeswitchSubjectIdx) 
-			t.Log(cfg.FreeswitchAccountIdx)
-			t.Log(cfg.FreeswitchDestIdx) 
-			t.Log(cfg.FreeswitchTimeStartIdx)
-			t.Log(cfg.FreeswitchDurationIdx)
-			t.Log(cfg.FreeswitchUUIDIdx)
-			t.Error("Config file read failed!")
-		}
 }
 
-func TestParamOverwrite(t *testing.T) {
-	cfgPth := "test_data.txt"
-	cfg, err := NewCGRConfig(&cfgPth)
-	if err != nil {
-		t.Log(fmt.Sprintf("Could not parse config: %s!", err))
-		t.FailNow()
+func TestLoadCgrCfgWithDefaults(t *testing.T) {
+	JSN_CFG := `
+{
+"sm_freeswitch": {
+	"enabled": true,				// starts SessionManager service: <true|false>
+	"connections":[					// instantiate connections to multiple FreeSWITCH servers
+		{"server": "1.2.3.4:8021", "password": "ClueCon", "reconnects": 3},
+		{"server": "1.2.3.5:8021", "password": "ClueCon", "reconnects": 5}
+	],
+},
+
+}`
+	eCgrCfg, _ := NewDefaultCGRConfig()
+	eCgrCfg.SmFsConfig.Enabled = true
+	eCgrCfg.SmFsConfig.Connections = []*FsConnConfig{
+		&FsConnConfig{Server: "1.2.3.4:8021", Password: "ClueCon", Reconnects: 3},
+		&FsConnConfig{Server: "1.2.3.5:8021", Password: "ClueCon", Reconnects: 5},
 	}
-	if cfg.FreeswitchReconnects != 5 { // one default which is not overwritten in test data
-		t.Errorf("FreeswitchReconnects set == %d, expect 5", cfg.FreeswitchReconnects)
-	} else if cfg.SchedulerEnabled != true { // one parameter which should be overwritten in test data
-		t.Errorf("scheduler_enabled set == %d, expect true", cfg.SchedulerEnabled)
+	if cgrCfg, err := NewCGRConfigFromJsonStringWithDefaults(JSN_CFG); err != nil {
+		t.Error(err)
+	} else if !reflect.DeepEqual(eCgrCfg.SmFsConfig, cgrCfg.SmFsConfig) {
+		t.Errorf("Expected: %+v, received: %+v", eCgrCfg.SmFsConfig, cgrCfg.SmFsConfig)
 	}
 }

config/configcdrc_test.go

@@ -0,0 +1,194 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) 2012-2015 ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/cgrates/cgrates/utils"
+)
+
+func TestLoadCdrcConfigMultipleFiles(t *testing.T) {
+	cgrCfg, err := NewCGRConfigFromFolder(".")
+	if err != nil {
+		t.Error(err)
+	}
+	eCgrCfg, _ := NewDefaultCGRConfig()
+	eCgrCfg.CdrcProfiles = make(map[string]map[string]*CdrcConfig)
+	// Default instance first
+	eCgrCfg.CdrcProfiles["/var/log/cgrates/cdrc/in"] = map[string]*CdrcConfig{
+		"*default": &CdrcConfig{
+			Enabled:                 false,
+			Cdrs:                    "internal",
+			CdrFormat:               "csv",
+			FieldSeparator:          ',',
+			DataUsageMultiplyFactor: 1024,
+			RunDelay:                0,
+			MaxOpenFiles:            1024,
+			CdrInDir:                "/var/log/cgrates/cdrc/in",
+			CdrOutDir:               "/var/log/cgrates/cdrc/out",
+			FailedCallsPrefix:       "missed_calls",
+			CdrSourceId:             "freeswitch_csv",
+			CdrFilter:               utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP),
+			PartialRecordCache:      time.Duration(10) * time.Second,
+			HeaderFields:            make([]*CfgCdrField, 0),
+			ContentFields: []*CfgCdrField{
+				&CfgCdrField{Tag: "tor", Type: "cdrfield", CdrFieldId: "tor", Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "accid", Type: "cdrfield", CdrFieldId: "accid", Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "reqtype", Type: "cdrfield", CdrFieldId: "reqtype", Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "direction", Type: "cdrfield", CdrFieldId: "direction", Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "tenant", Type: "cdrfield", CdrFieldId: "tenant", Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "category", Type: "cdrfield", CdrFieldId: "category", Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "account", Type: "cdrfield", CdrFieldId: "account", Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "subject", Type: "cdrfield", CdrFieldId: "subject", Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: utils.ParseRSRFieldsMustCompile("10", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "setup_time", Type: "cdrfield", CdrFieldId: "setup_time", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "answer_time", Type: "cdrfield", CdrFieldId: "answer_time", Value: utils.ParseRSRFieldsMustCompile("12", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "usage", Type: "cdrfield", CdrFieldId: "usage", Value: utils.ParseRSRFieldsMustCompile("13", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+			},
+			TrailerFields: make([]*CfgCdrField, 0),
+		},
+	}
+	eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc1/in"] = map[string]*CdrcConfig{
+		"CDRC-CSV1": &CdrcConfig{
+			Enabled:                 true,
+			Cdrs:                    "internal",
+			CdrFormat:               "csv",
+			FieldSeparator:          ',',
+			DataUsageMultiplyFactor: 1024,
+			RunDelay:                0,
+			MaxOpenFiles:            1024,
+			CdrInDir:                "/tmp/cgrates/cdrc1/in",
+			CdrOutDir:               "/tmp/cgrates/cdrc1/out",
+			CdrSourceId:             "csv1",
+			CdrFilter:               utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP),
+			HeaderFields:            make([]*CfgCdrField, 0),
+			ContentFields: []*CfgCdrField{
+				&CfgCdrField{Tag: "tor", Type: "cdrfield", CdrFieldId: "tor", Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "accid", Type: "cdrfield", CdrFieldId: "accid", Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "reqtype", Type: "cdrfield", CdrFieldId: "reqtype", Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "direction", Type: "cdrfield", CdrFieldId: "direction", Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "tenant", Type: "cdrfield", CdrFieldId: "tenant", Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "category", Type: "cdrfield", CdrFieldId: "category", Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "account", Type: "cdrfield", CdrFieldId: "account", Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "subject", Type: "cdrfield", CdrFieldId: "subject", Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: utils.ParseRSRFieldsMustCompile("10", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "setup_time", Type: "cdrfield", CdrFieldId: "setup_time", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "answer_time", Type: "cdrfield", CdrFieldId: "answer_time", Value: utils.ParseRSRFieldsMustCompile("12", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "usage", Type: "cdrfield", CdrFieldId: "usage", Value: utils.ParseRSRFieldsMustCompile("13", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+			},
+			TrailerFields: make([]*CfgCdrField, 0),
+		},
+	}
+	eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc2/in"] = map[string]*CdrcConfig{
+		"CDRC-CSV2": &CdrcConfig{
+			Enabled:                 true,
+			Cdrs:                    "internal",
+			CdrFormat:               "csv",
+			FieldSeparator:          ',',
+			DataUsageMultiplyFactor: 0.000976563,
+			RunDelay:                0,
+			MaxOpenFiles:            1024,
+			CdrInDir:                "/tmp/cgrates/cdrc2/in",
+			CdrOutDir:               "/tmp/cgrates/cdrc2/out",
+			CdrSourceId:             "csv2",
+			CdrFilter:               utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP),
+			HeaderFields:            make([]*CfgCdrField, 0),
+			ContentFields: []*CfgCdrField{
+				&CfgCdrField{Tag: "", Type: "", CdrFieldId: "tor", Value: utils.ParseRSRFieldsMustCompile("~7:s/^(voice|data|sms|generic)$/*$1/", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: false},
+				&CfgCdrField{Tag: "", Type: "", CdrFieldId: "answer_time", Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: false},
+			},
+			TrailerFields: make([]*CfgCdrField, 0),
+		},
+	}
+	eCgrCfg.CdrcProfiles["/tmp/cgrates/cdrc3/in"] = map[string]*CdrcConfig{
+		"CDRC-CSV3": &CdrcConfig{
+			Enabled:                 true,
+			Cdrs:                    "internal",
+			CdrFormat:               "csv",
+			FieldSeparator:          ',',
+			DataUsageMultiplyFactor: 1024,
+			RunDelay:                0,
+			MaxOpenFiles:            1024,
+			CdrInDir:                "/tmp/cgrates/cdrc3/in",
+			CdrOutDir:               "/tmp/cgrates/cdrc3/out",
+			CdrSourceId:             "csv3",
+			CdrFilter:               utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP),
+			HeaderFields:            make([]*CfgCdrField, 0),
+			ContentFields: []*CfgCdrField{
+				&CfgCdrField{Tag: "tor", Type: "cdrfield", CdrFieldId: "tor", Value: utils.ParseRSRFieldsMustCompile("2", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "accid", Type: "cdrfield", CdrFieldId: "accid", Value: utils.ParseRSRFieldsMustCompile("3", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "reqtype", Type: "cdrfield", CdrFieldId: "reqtype", Value: utils.ParseRSRFieldsMustCompile("4", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "direction", Type: "cdrfield", CdrFieldId: "direction", Value: utils.ParseRSRFieldsMustCompile("5", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "tenant", Type: "cdrfield", CdrFieldId: "tenant", Value: utils.ParseRSRFieldsMustCompile("6", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "category", Type: "cdrfield", CdrFieldId: "category", Value: utils.ParseRSRFieldsMustCompile("7", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "account", Type: "cdrfield", CdrFieldId: "account", Value: utils.ParseRSRFieldsMustCompile("8", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "subject", Type: "cdrfield", CdrFieldId: "subject", Value: utils.ParseRSRFieldsMustCompile("9", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "destination", Type: "cdrfield", CdrFieldId: "destination", Value: utils.ParseRSRFieldsMustCompile("10", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "setup_time", Type: "cdrfield", CdrFieldId: "setup_time", Value: utils.ParseRSRFieldsMustCompile("11", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "answer_time", Type: "cdrfield", CdrFieldId: "answer_time", Value: utils.ParseRSRFieldsMustCompile("12", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+				&CfgCdrField{Tag: "usage", Type: "cdrfield", CdrFieldId: "usage", Value: utils.ParseRSRFieldsMustCompile("13", utils.INFIELD_SEP),
+					FieldFilter: utils.ParseRSRFieldsMustCompile("", utils.INFIELD_SEP), Width: 0, Strip: "", Padding: "", Layout: "", Mandatory: true},
+			},
+			TrailerFields: make([]*CfgCdrField, 0),
+		},
+	}
+	if !reflect.DeepEqual(eCgrCfg.CdrcProfiles, cgrCfg.CdrcProfiles) {
+		t.Errorf("Expected: %+v, received: %+v", eCgrCfg.CdrcProfiles, cgrCfg.CdrcProfiles)
+	}
+}

config/libconfig.go

@@ -0,0 +1,30 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"github.com/cgrates/cgrates/utils"
+)
+
+type CdrReplicationCfg struct {
+	Transport   string
+	Server      string
+	Synchronous bool
+	CdrFilter   utils.RSRFields // Only replicate if the filters here are matching
+}

config/libconfig_json.go

@@ -0,0 +1,253 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+// General config section
+type GeneralJsonCfg struct {
+	Http_skip_tls_verify *bool
+	Rounding_decimals    *int
+	Dbdata_encoding      *string
+	Tpexport_dir         *string
+	Default_reqtype      *string
+	Default_category     *string
+	Default_tenant       *string
+	Default_subject      *string
+	Default_timezone     *string
+	Reconnects           *int
+	Connect_attempts     *int
+	Response_cache_ttl   *string
+	Internal_ttl         *string
+}
+
+// Listen config section
+type ListenJsonCfg struct {
+	Rpc_json *string
+	Rpc_gob  *string
+	Http     *string
+}
+
+// Database config
+type DbJsonCfg struct {
+	Db_type           *string
+	Db_host           *string
+	Db_port           *int
+	Db_name           *string
+	Db_user           *string
+	Db_passwd         *string
+	Max_open_conns    *int // Used only in case of storDb
+	Max_idle_conns    *int
+	Load_history_size *int // Used in case of dataDb to limit the length of the loads history
+}
+
+// Balancer config section
+type BalancerJsonCfg struct {
+	Enabled *bool
+}
+
+// Rater config section
+type RaterJsonCfg struct {
+	Enabled  *bool
+	Balancer *string
+	Cdrstats *string
+	Historys *string
+	Pubsubs  *string
+	Aliases  *string
+	Users    *string
+}
+
+// Scheduler config section
+type SchedulerJsonCfg struct {
+	Enabled *bool
+}
+
+// Cdrs config section
+type CdrsJsonCfg struct {
+	Enabled         *bool
+	Extra_fields    *[]string
+	Store_cdrs      *bool
+	Rater           *string
+	Pubsubs         *string
+	Users           *string
+	Aliases         *string
+	Cdrstats        *string
+	Cdr_replication *[]*CdrReplicationJsonCfg
+}
+
+type CdrReplicationJsonCfg struct {
+	Transport   *string
+	Server      *string
+	Synchronous *bool
+	Cdr_filter  *string
+}
+
+// Cdrstats config section
+type CdrStatsJsonCfg struct {
+	Enabled       *bool
+	Save_Interval *string
+}
+
+// One cdr field config, used in cdre and cdrc
+type CdrFieldJsonCfg struct {
+	Tag          *string
+	Type         *string
+	Cdr_field_id *string
+	Metatag_id   *string
+	Value        *string
+	Width        *int
+	Strip        *string
+	Padding      *string
+	Layout       *string
+	Field_filter *string
+	Mandatory    *bool
+}
+
+// Cdre config section
+type CdreJsonCfg struct {
+	Cdr_format                    *string
+	Field_separator               *string
+	Data_usage_multiply_factor    *float64
+	Sms_usage_multiply_factor     *float64
+	Generic_usage_multiply_factor *float64
+	Cost_multiply_factor          *float64
+	Cost_rounding_decimals        *int
+	Cost_shift_digits             *int
+	Mask_destination_id           *string
+	Mask_length                   *int
+	Export_dir                    *string
+	Header_fields                 *[]*CdrFieldJsonCfg
+	Content_fields                *[]*CdrFieldJsonCfg
+	Trailer_fields                *[]*CdrFieldJsonCfg
+}
+
+// Cdrc config section
+type CdrcJsonCfg struct {
+	Enabled                    *bool
+	Dry_run                    *bool
+	Cdrs                       *string
+	Cdr_format                 *string
+	Field_separator            *string
+	Timezone                   *string
+	Run_delay                  *int
+	Data_usage_multiply_factor *float64
+	Cdr_in_dir                 *string
+	Cdr_out_dir                *string
+	Failed_calls_prefix        *string
+	Cdr_source_id              *string
+	Cdr_filter                 *string
+	Max_open_files             *int
+	Partial_record_cache       *string
+	Header_fields              *[]*CdrFieldJsonCfg
+	Content_fields             *[]*CdrFieldJsonCfg
+	Trailer_fields             *[]*CdrFieldJsonCfg
+}
+
+// SM-FreeSWITCH config section
+type SmFsJsonCfg struct {
+	Enabled                *bool
+	Rater                  *string
+	Cdrs                   *string
+	Create_cdr             *bool
+	Extra_fields           *[]string
+	Debit_interval         *string
+	Min_call_duration      *string
+	Max_call_duration      *string
+	Min_dur_low_balance    *string
+	Low_balance_ann_file   *string
+	Empty_balance_context  *string
+	Empty_balance_ann_file *string
+	Subscribe_park         *bool
+	Channel_sync_interval  *string
+	Connections            *[]*FsConnJsonCfg
+}
+
+// Represents one connection instance towards FreeSWITCH
+type FsConnJsonCfg struct {
+	Server     *string
+	Password   *string
+	Reconnects *int
+}
+
+// SM-Kamailio config section
+type SmKamJsonCfg struct {
+	Enabled           *bool
+	Rater             *string
+	Cdrs              *string
+	Create_cdr        *bool
+	Debit_interval    *string
+	Min_call_duration *string
+	Max_call_duration *string
+	Connections       *[]*KamConnJsonCfg
+}
+
+// Represents one connection instance towards Kamailio
+type KamConnJsonCfg struct {
+	Evapi_addr *string
+	Reconnects *int
+}
+
+// SM-OpenSIPS config section
+type SmOsipsJsonCfg struct {
+	Enabled                   *bool
+	Listen_udp                *string
+	Rater                     *string
+	Cdrs                      *string
+	Create_cdr                *bool
+	Debit_interval            *string
+	Min_call_duration         *string
+	Max_call_duration         *string
+	Events_subscribe_interval *string
+	Mi_addr                   *string
+}
+
+// Represents one connection instance towards OpenSIPS
+type OsipsConnJsonCfg struct {
+	Mi_addr    *string
+	Reconnects *int
+}
+
+// History server config section
+type HistServJsonCfg struct {
+	Enabled       *bool
+	History_dir   *string
+	Save_interval *string
+}
+
+// PubSub server config section
+type PubSubServJsonCfg struct {
+	Enabled *bool
+}
+
+// Aliases server config section
+type AliasesServJsonCfg struct {
+	Enabled *bool
+}
+
+// Users server config section
+type UserServJsonCfg struct {
+	Enabled *bool
+	Indexes *[]string
+}
+
+// Mailer config section
+type MailerJsonCfg struct {
+	Server       *string
+	Auth_user    *string
+	Auth_passwd  *string
+	From_address *string
+}

config/multifiles_local_test.go

@@ -0,0 +1,106 @@
+/*
+Real-time Charging System for Telecom & ISP environments
+Copyright (C) ITsysCOM GmbH
+
+This program is free software: you can Storagetribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITH*out ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>
+*/
+
+package config
+
+import (
+	"flag"
+	"github.com/cgrates/cgrates/utils"
+	"testing"
+)
+
+var testLocal = flag.Bool("local", false, "Perform the tests only on local test environment, disabled by default.") // This flag will be passed here via "go test -local" args
+
+var mfCgrCfg *CGRConfig
+
+func TestMfInitConfig(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	var err error
+	if mfCgrCfg, err = NewCGRConfigFromFolder("/usr/share/cgrates/conf/samples/multifiles"); err != nil {
+		t.Fatal("Got config error: ", err.Error())
+	}
+}
+
+func TestMfGeneralItems(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	if mfCgrCfg.DefaultReqType != utils.META_PSEUDOPREPAID { // Twice reconfigured
+		t.Error("DefaultReqType: ", mfCgrCfg.DefaultReqType)
+	}
+	if mfCgrCfg.DefaultCategory != "call" { // Not configred, should be inherited from default
+		t.Error("DefaultCategory: ", mfCgrCfg.DefaultCategory)
+	}
+}
+
+func TestMfCdreDefaultInstance(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	for _, prflName := range []string{"*default", "export1"} {
+		if _, hasIt := mfCgrCfg.CdreProfiles[prflName]; !hasIt {
+			t.Error("Cdre does not contain profile ", prflName)
+		}
+	}
+	prfl := "*default"
+	if mfCgrCfg.CdreProfiles[prfl].CdrFormat != "csv" {
+		t.Error("Default instance has cdrFormat: ", mfCgrCfg.CdreProfiles[prfl].CdrFormat)
+	}
+	if mfCgrCfg.CdreProfiles[prfl].DataUsageMultiplyFactor != 1024.0 {
+		t.Error("Default instance has cdrFormat: ", mfCgrCfg.CdreProfiles[prfl].DataUsageMultiplyFactor)
+	}
+	if len(mfCgrCfg.CdreProfiles[prfl].HeaderFields) != 0 {
+		t.Error("Default instance has number of header fields: ", len(mfCgrCfg.CdreProfiles[prfl].HeaderFields))
+	}
+	if len(mfCgrCfg.CdreProfiles[prfl].ContentFields) != 12 {
+		t.Error("Default instance has number of content fields: ", len(mfCgrCfg.CdreProfiles[prfl].ContentFields))
+	}
+	if mfCgrCfg.CdreProfiles[prfl].ContentFields[2].Tag != "Direction" {
+		t.Error("Unexpected headerField value: ", mfCgrCfg.CdreProfiles[prfl].ContentFields[2].Tag)
+	}
+}
+
+func TestMfCdreExport1Instance(t *testing.T) {
+	if !*testLocal {
+		return
+	}
+	prfl := "export1"
+	if mfCgrCfg.CdreProfiles[prfl].CdrFormat != "csv" {
+		t.Error("Export1 instance has cdrFormat: ", mfCgrCfg.CdreProfiles[prfl].CdrFormat)
+	}
+	if mfCgrCfg.CdreProfiles[prfl].DataUsageMultiplyFactor != 1.0 {
+		t.Error("Export1 instance has DataUsageMultiplyFormat: ", mfCgrCfg.CdreProfiles[prfl].DataUsageMultiplyFactor)
+	}
+	if mfCgrCfg.CdreProfiles[prfl].CostRoundingDecimals != 3.0 {
+		t.Error("Export1 instance has cdrFormat: ", mfCgrCfg.CdreProfiles[prfl].CostRoundingDecimals)
+	}
+	if len(mfCgrCfg.CdreProfiles[prfl].HeaderFields) != 2 {
+		t.Error("Export1 instance has number of header fields: ", len(mfCgrCfg.CdreProfiles[prfl].HeaderFields))
+	}
+	if mfCgrCfg.CdreProfiles[prfl].HeaderFields[1].Tag != "RunId" {
+		t.Error("Unexpected headerField value: ", mfCgrCfg.CdreProfiles[prfl].HeaderFields[1].Tag)
+	}
+	if len(mfCgrCfg.CdreProfiles[prfl].ContentFields) != 9 {
+		t.Error("Export1 instance has number of content fields: ", len(mfCgrCfg.CdreProfiles[prfl].ContentFields))
+	}
+	if mfCgrCfg.CdreProfiles[prfl].ContentFields[2].Tag != "Account" {
+		t.Error("Unexpected headerField value: ", mfCgrCfg.CdreProfiles[prfl].ContentFields[2].Tag)
+	}
+}