Improve functionality for Drone CI

data/docker/integration/Dockerfile

@@ -1,65 +1,78 @@
-FROM  ubuntu:jammy
+FROM debian:bullseye-slim
 
 RUN groupadd -r mongodb && useradd -r -g mongodb mongodb
 RUN groupadd -r redis && useradd -r -g redis redis 
 RUN groupadd -r kafka && useradd -r -g kafka kafka 
+RUN groupadd -r nats && useradd -r -g nats nats
 
-RUN mkdir -p /usr/share/cgrates/conf /usr/share/man/man1 /var/spool/cgrates/ers/in /var/spool/cgrates/ers/out /var/spool/cgrates/cdre/csv /var/spool/cgrates/cdre/fwv /var/spool/cgrates/tpe /var/spool/cgrates/failed_posts /var/spool/cgrates/analyzers /run /data/configdb /data/db /kafka /logs
+RUN mkdir -p /usr/share/cgrates/conf /usr/share/man/man1 /var/spool/cgrates/ers/in /var/spool/cgrates/ers/out /var/spool/cgrates/cdre/csv /var/spool/cgrates/cdre/fwv /var/spool/cgrates/tpe /var/spool/cgrates/failed_posts /var/spool/cgrates/analyzers /run /data/configdb /data/db /kafka /logs /opt/nats
 
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-
-RUN apt-get update && apt-get install -y ansible
-
-# Install Erlang packages
-RUN apt-get install -y erlang-base erlang-asn1 erlang-crypto erlang-eldap erlang-ftp erlang-inets erlang-mnesia erlang-os-mon erlang-parsetools erlang-public-key erlang-runtime-tools erlang-snmp erlang-ssl erlang-syntax-tools erlang-tftp erlang-tools erlang-xmerl
-
-RUN apt-get install -y apt-utils wget gnupg gnupg2 apt-transport-https curl redis-server git build-essential rsyslog procps gosu "mariadb-server" mariadb-backup socat default-jdk-headless neovim net-tools
-
-RUN curl -fsSL https://pgp.mongodb.com/server-7.0.asc | gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg  --dearmor
-RUN echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-7.0.list
-
-RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
-RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc |  apt-key add -
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
 
 RUN apt-get update
-RUN apt-get install -y mongodb-org postgresql
-# ADD THIS AFTER postgresql rabbitmq-server
+
+RUN apt-get install -y apt-utils wget gnupg gnupg2 apt-transport-https curl redis-server git gcc build-essential rsyslog procps gosu "mariadb-server" mariadb-backup socat default-jdk neovim net-tools
+
+RUN wget https://github.com/nats-io/nats-server/releases/download/v2.9.17/nats-server-v2.9.17-linux-amd64.tar.gz -O /tmp/nats-server-v2.9.17-linux-amd64.tar.gz && tar -xzvf /tmp/nats-server-v2.9.17-linux-amd64.tar.gz -C /opt/nats --strip-components=1
+
+RUN ln -s /opt/nats/nats-server /usr/local/bin/nats-server
+
+RUN rm /tmp/nats-server-v2.9.17-linux-amd64.tar.gz
+
+RUN curl -fsSL https://pgp.mongodb.com/server-7.0.asc | gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg  --dearmor
+RUN echo "deb [ signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/debian bullseye/mongodb-org/7.0 main" | tee /etc/apt/sources.list.d/mongodb-org-7.0.list
+
+RUN curl -1sLf "https://keys.openpgp.org/vks/v1/by-fingerprint/0A9AF2115F4687BD29803A206B73A36E6026DFCA" | gpg --dearmor | tee /usr/share/keyrings/com.rabbitmq.team.gpg > /dev/null
+RUN curl -1sLf "https://github.com/rabbitmq/signing-keys/releases/download/3.0/cloudsmith.rabbitmq-erlang.E495BB49CC4BBE5B.key" | gpg --dearmor | tee /usr/share/keyrings/rabbitmq.E495BB49CC4BBE5B.gpg > /dev/null
+RUN curl -1sLf "https://github.com/rabbitmq/signing-keys/releases/download/3.0/cloudsmith.rabbitmq-server.9F4587F226208342.key" | gpg --dearmor | tee /usr/share/keyrings/rabbitmq.9F4587F226208342.gpg > /dev/null
+RUN tee /etc/apt/sources.list.d/rabbitmq.list <<EOF
+## Provides modern Erlang/OTP releases
+##
+deb [signed-by=/usr/share/keyrings/rabbitmq.E495BB49CC4BBE5B.gpg] https://ppa1.novemberain.com/rabbitmq/rabbitmq-erlang/deb/debian bullseye main
+deb-src [signed-by=/usr/share/keyrings/rabbitmq.E495BB49CC4BBE5B.gpg] https://ppa1.novemberain.com/rabbitmq/rabbitmq-erlang/deb/debian bullseye main
+
+# another mirror for redundancy
+deb [signed-by=/usr/share/keyrings/rabbitmq.E495BB49CC4BBE5B.gpg] https://ppa2.novemberain.com/rabbitmq/rabbitmq-erlang/deb/debian bullseye main
+deb-src [signed-by=/usr/share/keyrings/rabbitmq.E495BB49CC4BBE5B.gpg] https://ppa2.novemberain.com/rabbitmq/rabbitmq-erlang/deb/debian bullseye main
+
+## Provides RabbitMQ
+##
+deb [signed-by=/usr/share/keyrings/rabbitmq.9F4587F226208342.gpg] https://ppa1.novemberain.com/rabbitmq/rabbitmq-server/deb/debian bullseye main
+deb-src [signed-by=/usr/share/keyrings/rabbitmq.9F4587F226208342.gpg] https://ppa1.novemberain.com/rabbitmq/rabbitmq-server/deb/debian bullseye main
+
+# another mirror for redundancy
+deb [signed-by=/usr/share/keyrings/rabbitmq.9F4587F226208342.gpg] https://ppa2.novemberain.com/rabbitmq/rabbitmq-server/deb/debian bullseye main
+deb-src [signed-by=/usr/share/keyrings/rabbitmq.9F4587F226208342.gpg] https://ppa2.novemberain.com/rabbitmq/rabbitmq-server/deb/debian bullseye main
+EOF
+
+RUN apt-get update
+RUN apt-get install -y erlang-base erlang-asn1 erlang-crypto erlang-eldap erlang-ftp erlang-inets erlang-mnesia erlang-os-mon erlang-parsetools erlang-public-key erlang-runtime-tools erlang-snmp erlang-ssl erlang-syntax-tools erlang-tftp erlang-tools erlang-xmerl
+RUN apt-get install -y  rabbitmq-server mongodb-org postgresql --fix-missing
 
 WORKDIR /kafka
 
-RUN wget "https://archive.apache.org/dist/kafka/2.1.0/kafka_2.11-2.1.0.tgz" 
-RUN tar -xvzf ./kafka_2.11-2.1.0.tgz --strip 1
-RUN rm kafka_2.11-2.1.0.tgz
+RUN wget "https://dlcdn.apache.org/kafka/3.5.0/kafka_2.13-3.5.0.tgz" 
+RUN tar -xvzf ./kafka_2.13-3.5.0.tgz --strip 1
+RUN rm kafka_2.13-3.5.0.tgz
 COPY ./conf/server.properties /kafka/config/server.properties
 WORKDIR /
 
 RUN set -ex; \
 	rm -rf /var/lib/apt/lists/*; \
-# purge and re-create /var/lib/mysql with appropriate ownership
 	rm -rf /var/lib/mysql; \
 	mkdir -p /var/lib/mysql /var/run/mysqld; \
 	chown -R mysql:mysql /var/lib/mysql /var/run/mysqld; \
-# ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instance ends up having at runtime
 	chmod 777 /var/run/mysqld; \
-# comment out a few problematic configuration values
 	find /etc/mysql/ -name '*.cnf' -print0 \
 		| xargs -0 grep -lZE '^(bind-address|log|user\s)' \
 		| xargs -rt -0 sed -Ei 's/^(bind-address|log|user\s)/#&/'; \
-# don't reverse lookup hostnames, they are usually another container
 	echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf
 
-# Clean up
 RUN apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
 RUN touch /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log 
-RUN chmod 777  /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log 
-
-COPY main.yaml /integration_tests.yaml
-
-COPY roles /roles
+RUN chmod 777  /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log
 
 RUN wget -O go.tgz "https://storage.googleapis.com/golang/go1.21.0.linux-amd64.tar.gz" --progress=dot:giga
 RUN tar -C /usr/local -xzf go.tgz

data/docker/integration/docker-entrypoint.sh

@@ -1,7 +1,9 @@
 #!/bin/bash
 set -ev
 
-ansible-playbook /integration_tests.yaml -i localhost, -c local -e "ansible_user=root clone_repository=true" -vvv
+export MYSQL_ROOT_PASSWORD="CGRateS.org"
+
+/go/src/github.com/cgrates/cgrates/build.sh
 
 # # Create symbolic links
 ln -s "/go/src/github.com/cgrates/cgrates/data" "/usr/share/cgrates"
@@ -13,61 +15,58 @@ ln -s "/go/bin/cgr-tester" "/usr/bin/cgr-tester"
 
 # start basic subsystems
 # export KAFKA_HEAP_OPTS="-Xmx100M -Xms100M"
-# /kafka/bin/zookeeper-server-start.sh -daemon /kafka/config/zookeeper.properties
-# /kafka/bin/kafka-server-start.sh -daemon /kafka/config/server.properties
-
+/kafka/bin/zookeeper-server-start.sh -daemon /kafka/config/zookeeper.properties
+/kafka/bin/kafka-server-start.sh -daemon /kafka/config/server.properties
 rsyslogd -f /etc/rsyslogd.conf 
 version=$(ls /var/lib/postgresql)
 pg_ctlcluster $version main start &
 mongod --bind_ip 127.0.0.1  --logpath /logs/mongodb.log &
 redis-server /etc/redis/redis.conf &
-MYSQL_ROOT_PASSWORD="CGRateS.org" /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mariadb-ep.sh mysqld
-# rabbitmq-server > /logs/rabbitmq.log  2>&1 &
+/go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mariadb-ep.sh mysqld
+rabbitmq-server > /logs/rabbitmq.log  2>&1 &
 
 
-# START_TIMEOUT=600
+START_TIMEOUT=600
 
-# start_timeout_exceeded=false
-# count=0
-# step=10
-# while netstat -lnt | awk '$4 ~ /:9092$/ {exit 1}'; do
-#     echo "waiting for kafka to be ready"
-#     sleep $step;
-#     count=$((count + step))
-#     if [ $count -gt $START_TIMEOUT ]; then
-#         start_timeout_exceeded=true
-#         break
-#     fi
-# done
+start_timeout_exceeded=false
+count=0
+step=10
+while netstat -lnt | awk '$4 ~ /:9092$/ {exit 1}'; do
+    echo "waiting for kafka to be ready"
+    sleep $step;
+    count=$((count + step))
+    if [ $count -gt $START_TIMEOUT ]; then
+        start_timeout_exceeded=true
+        break
+    fi
+done
 
-# if $start_timeout_exceeded; then
-#     echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)"
-#     exit 1
-# fi
+if $start_timeout_exceeded; then
+    echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)"
+    exit 1
+fi
 
-# /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates
-# /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates_cdrs
+/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic cgrates
+/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic cgrates_cdrs
+
+
+gosu postgres psql  -c "CREATE USER cgrates password 'CGRateS.org';"  
+gosu postgres createdb -e -O cgrates cgrates 
+gosu postgres createdb -e -O cgrates cgrates2
 
 
 
-# gosu postgres psql  -c "CREATE USER cgrates password 'CGRateS.org';"  > /dev/null 2>&1
-# gosu postgres createdb -e -O cgrates cgrates > /dev/null 2>&1
+PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/postgres/create_cdrs_tables.sql 
+PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/postgres/create_tariffplan_tables.sql 
 
 
-
-# PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /cgrates/data/docker/integration/scripts/postgres/create_cdrs_tables.sql >/dev/null 2>&1
-# PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /cgrates/data/docker/integration/scripts/postgres/create_tariffplan_tables.sql >/dev/null 2>&1
-
-
-# mongo --quiet /cgrates/data/docker/integration/scripts/create_user.js >/dev/null 2>&1
-echo "Starting..."
-sleep 60  # Pause for 1 min
-echo "Resuming..."
+mongosh --quiet /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mongo/create_user.js
 mysql -u root -pCGRateS.org -h localhost < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_db_with_users.sql 
+mysql -u root -pCGRateS.org -h localhost < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_ers_db.sql
 mysql -u root -pCGRateS.org -h localhost -D cgrates < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_cdrs_tables.sql
 mysql -u root -pCGRateS.org -h localhost -D cgrates < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_tariffplan_tables.sql
 
 cp -r /go/src/github.com/cgrates/cgrates/data/. /usr/share/cgrates
 
 # Set versions
-# cgr-migrator -exec=*set_versions -config_path=/usr/share/cgrates/conf/samples/tutmysql
+cgr-migrator -exec=*set_versions -config_path=/usr/share/cgrates/conf/samples/tutmysql

data/docker/integration/main.yaml

@@ -1,46 +0,0 @@
----
-- hosts: all
-  vars:
-    golang_gopath: "/go"
-    clone_repository: true
-    cgrates_dir: "{{ golang_gopath }}/src/github.com/cgrates/cgrates"
-    golang_install_dir: /usr/local/go
-    git_version: "master"
-    cgrates_dependencies:
-      - git
-      - redis-server
-      - mariadb-server
-      - make
-      - gcc
-
-  tasks:
-    - name: Install CGRateS dependencies
-      become: yes
-      ansible.builtin.package:
-        name: "{{ cgrates_dependencies }}"
-        state: present
-        update_cache: yes
-        cache_valid_time: 86400
-
-    - name: Create cgrates directory
-      ansible.builtin.file:
-        state: directory
-        mode: "u=rwx,go=rx"
-        owner: "{{ ansible_user }}"
-        dest: "{{ cgrates_dir }}"
-      when: clone_repository | bool
-
-    - name: Git clone cgrates
-      ansible.builtin.git:
-        repo: https://github.com/cgrates/cgrates.git
-        dest: "{{ cgrates_dir }}"
-        update: yes
-        force: yes
-        version: "{{ git_version }}"
-      when: clone_repository | bool
-
-    - name: Build cgrates
-      ansible.builtin.shell:
-        cmd: bash -lc "sh {{ cgrates_dir }}/build.sh"
-      args:
-        chdir: "{{ cgrates_dir }}"

data/docker/integration/scripts/mongo

@@ -1 +0,0 @@
-../../../storage/mongo/

data/docker/integration/scripts/mongo/create_user.js

@@ -0,0 +1,9 @@
+db = db.getSiblingDB('cgrates')
+db.createUser(
+  {
+    user: "cgrates",
+    pwd: "CGRateS.org",
+    roles: [ { role: "dbAdmin", db: "cgrates" } ]
+  }
+)
+

data/docker/integration/scripts/mysql

@@ -1 +0,0 @@
-../../../storage/mysql/

data/docker/integration/scripts/mysql/alter_cdr_tables_rc5_rc6.sql

@@ -0,0 +1,50 @@
+USE `cgrates`;
+
+ALTER TABLE `cdrs_primary` 
+	CHANGE COLUMN tbid `id` int(11) NOT NULL auto_increment first ,
+	CHANGE `cgrid` `cgrid` char(40) NOT NULL after `id` ,
+	ADD COLUMN `pdd` decimal(12,9) NOT NULL after `setup_time` ,
+	CHANGE `answer_time` `answer_time` datetime  NULL after `pdd` ,
+	ADD COLUMN `supplier` varchar(128) NOT NULL after `usage` , 
+	ADD COLUMN `disconnect_cause` varchar(64) NOT NULL after `supplier` , 
+	ADD COLUMN `created_at` timestamp   NOT NULL DEFAULT CURRENT_TIMESTAMP  on update CURRENT_TIMESTAMP after `disconnect_cause` , 
+	ADD COLUMN `deleted_at` timestamp   NOT NULL DEFAULT '0000-00-00 00:00:00' after `created_at` ,
+	ADD KEY `answer_time_idx`(`answer_time`) , 
+	ADD KEY `deleted_at_idx`(`deleted_at`) , 
+	DROP KEY `PRIMARY`, ADD PRIMARY KEY(`id`) ;
+
+ALTER TABLE `cdrs_extra` 
+	CHANGE COLUMN tbid `id` int(11) NOT NULL auto_increment first ,
+	CHANGE `cgrid` `cgrid` char(40) NOT NULL after `id` , 
+	ADD COLUMN `created_at` timestamp   NOT NULL DEFAULT CURRENT_TIMESTAMP  on update CURRENT_TIMESTAMP after `extra_fields` , 
+	ADD COLUMN `deleted_at` timestamp   NOT NULL DEFAULT '0000-00-00 00:00:00' after `created_at`,
+	ADD UNIQUE KEY `cgrid`(`cgrid`) , 
+	ADD KEY `deleted_at_idx`(`deleted_at`) , 
+	DROP KEY `PRIMARY`, ADD PRIMARY KEY(`id`) ;
+
+ALTER TABLE `cost_details`
+	CHANGE COLUMN tbid `id` int(11) NOT NULL auto_increment first ,
+	CHANGE `cost_source` `cost_source` varchar(64) NOT NULL after `timespans` , 
+	ADD COLUMN `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP after `cost_source` , 
+	ADD COLUMN `updated_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' after `created_at` , 
+	ADD COLUMN `deleted_at` timestamp   NOT NULL DEFAULT '0000-00-00 00:00:00' after `updated_at` , 
+	DROP COLUMN `cost_time` , 
+	ADD KEY `deleted_at_idx`(`deleted_at`) , 
+	DROP KEY `PRIMARY`, ADD PRIMARY KEY(`id`) ;
+
+ALTER TABLE `rated_cdrs` 
+	CHANGE COLUMN tbid `id` int(11) NOT NULL auto_increment first ,
+	CHANGE `cgrid` `cgrid` char(40) NOT NULL after `id` , 
+	CHANGE `category` `category` varchar(32) NOT NULL after `tenant` , 
+	ADD COLUMN `pdd` decimal(12,9)   NOT NULL after `setup_time` , 
+	CHANGE `answer_time` `answer_time` datetime  NULL after `pdd` , 
+	ADD COLUMN `supplier` varchar(128) NOT NULL after `usage` , 
+	ADD COLUMN `disconnect_cause` varchar(64) NOT NULL after `supplier` , 
+	CHANGE `cost` `cost` decimal(20,4)   NULL after `disconnect_cause` , 
+	ADD COLUMN `created_at` timestamp   NOT NULL DEFAULT CURRENT_TIMESTAMP  on update CURRENT_TIMESTAMP after `extra_info` , 
+	ADD COLUMN `updated_at` timestamp   NOT NULL DEFAULT '0000-00-00 00:00:00' after `created_at` , 
+	ADD COLUMN `deleted_at` timestamp   NOT NULL DEFAULT '0000-00-00 00:00:00' after `updated_at` , 
+	DROP COLUMN `mediation_time` , 
+	ADD KEY `deleted_at_idx`(`deleted_at`) , 
+	DROP KEY `PRIMARY`, ADD PRIMARY KEY(`id`) ;
+

data/docker/integration/scripts/mysql/create_cdrs_tables.sql

@@ -0,0 +1,52 @@
+--
+-- Table structure for table `cdrs`
+--
+
+DROP TABLE IF EXISTS cdrs;
+CREATE TABLE cdrs (
+  id int(11) NOT NULL AUTO_INCREMENT,
+  cgrid varchar(40) NOT NULL,
+  run_id  varchar(64) NOT NULL,
+  origin_host varchar(64) NOT NULL,
+  source varchar(64) NOT NULL,
+  origin_id varchar(128) NOT NULL,
+  tor  varchar(16) NOT NULL,
+  request_type varchar(24) NOT NULL,
+  tenant varchar(64) NOT NULL,
+  category varchar(64) NOT NULL,
+  account varchar(128) NOT NULL,
+  subject varchar(128) NOT NULL,
+  destination varchar(128) NOT NULL,
+  setup_time datetime NOT NULL,
+  answer_time datetime NULL,
+  `usage` BIGINT NOT NULL,
+  extra_fields text NOT NULL,
+  cost_source varchar(64) NOT NULL,
+  cost DECIMAL(20,4) NOT NULL,
+  cost_details MEDIUMTEXT,
+  extra_info text,
+  created_at TIMESTAMP NULL,
+  updated_at TIMESTAMP NULL,
+  deleted_at TIMESTAMP NULL,
+  PRIMARY KEY (id),
+  UNIQUE KEY cdrrun (cgrid, run_id)
+);
+
+DROP TABLE IF EXISTS session_costs;
+CREATE TABLE session_costs (
+  id int(11) NOT NULL AUTO_INCREMENT,
+  cgrid varchar(40) NOT NULL,
+  run_id  varchar(64) NOT NULL,
+  origin_host varchar(64) NOT NULL,
+  origin_id varchar(128) NOT NULL,
+  cost_source varchar(64) NOT NULL,
+  `usage` BIGINT NOT NULL,
+  cost_details MEDIUMTEXT,
+  created_at TIMESTAMP NULL,
+  deleted_at TIMESTAMP NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY costid (cgrid, run_id),
+  KEY origin_idx (origin_host, origin_id),
+  KEY run_origin_idx (run_id, origin_id),
+  KEY deleted_at_idx (deleted_at)
+);

data/docker/integration/scripts/mysql/create_db_with_users.sql

@@ -0,0 +1,10 @@
+
+--
+-- Sample db and users creation. Replace here with your own details
+--
+
+DROP DATABASE IF EXISTS cgrates;
+CREATE DATABASE cgrates;
+CREATE USER IF NOT EXISTS 'cgrates'@'127.0.0.1' IDENTIFIED BY 'CGRateS.org';
+GRANT ALL PRIVILEGES ON cgrates.* TO 'cgrates'@'127.0.0.1' WITH GRANT OPTION;
+FLUSH PRIVILEGES;

data/docker/integration/scripts/mysql/create_ers_db.sql

@@ -0,0 +1,7 @@
+
+--
+-- extra DB for ees and ers
+DROP DATABASE IF EXISTS cgrates2;
+CREATE DATABASE cgrates2;
+
+GRANT ALL on cgrates2.* TO 'cgrates'@'127.0.0.1' IDENTIFIED BY 'CGRateS.org';

data/docker/integration/scripts/mysql/create_tariffplan_tables.sql

@@ -0,0 +1,491 @@
+--
+-- Table structure for table `tp_timings`
+--
+DROP TABLE IF EXISTS `tp_timings`;
+CREATE TABLE `tp_timings` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tag` varchar(64) NOT NULL,
+  `years` varchar(255) NOT NULL,
+  `months` varchar(255) NOT NULL,
+  `month_days` varchar(255) NOT NULL,
+  `week_days` varchar(255) NOT NULL,
+  `time` varchar(32) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `tpid` (`tpid`),
+  KEY `tpid_tmid` (`tpid`,`tag`),
+  UNIQUE KEY `tpid_tag` (`tpid`,`tag`)
+);
+
+--
+-- Table structure for table `tp_destinations`
+--
+
+DROP TABLE IF EXISTS `tp_destinations`;
+CREATE TABLE `tp_destinations` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tag` varchar(64) NOT NULL,
+  `prefix` varchar(24) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `tpid` (`tpid`),
+  KEY `tpid_dstid` (`tpid`,`tag`),
+  UNIQUE KEY `tpid_dest_prefix` (`tpid`,`tag`,`prefix`)
+);
+
+--
+-- Table structure for table `tp_rates`
+--
+
+DROP TABLE IF EXISTS `tp_rates`;
+CREATE TABLE `tp_rates` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tag` varchar(64) NOT NULL,
+  `connect_fee` decimal(7,4) NOT NULL,
+  `rate` decimal(10,4) NOT NULL,
+  `rate_unit` varchar(16) NOT NULL,
+  `rate_increment` varchar(16) NOT NULL,
+  `group_interval_start` varchar(16) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `unique_tprate` (`tpid`,`tag`,`group_interval_start`),
+  KEY `tpid` (`tpid`),
+  KEY `tpid_rtid` (`tpid`,`tag`)
+);
+
+--
+-- Table structure for table `destination_rates`
+--
+
+DROP TABLE IF EXISTS `tp_destination_rates`;
+CREATE TABLE `tp_destination_rates` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tag` varchar(64) NOT NULL,
+  `destinations_tag` varchar(64) NOT NULL,
+  `rates_tag` varchar(64) NOT NULL,
+  `rounding_method` varchar(255) NOT NULL,
+  `rounding_decimals` tinyint(4) NOT NULL,
+  `max_cost` decimal(7,4) NOT NULL,
+  `max_cost_strategy` varchar(16) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `tpid` (`tpid`),
+  KEY `tpid_drid` (`tpid`,`tag`),
+  UNIQUE KEY `tpid_drid_dstid` (`tpid`,`tag`,`destinations_tag`)
+);
+
+--
+-- Table structure for table `tp_rating_plans`
+--
+
+DROP TABLE IF EXISTS `tp_rating_plans`;
+CREATE TABLE `tp_rating_plans` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tag` varchar(64) NOT NULL,
+  `destrates_tag` varchar(64) NOT NULL,
+  `timing_tag` varchar(64) NOT NULL,
+  `weight` DECIMAL(8,2) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `tpid` (`tpid`),
+  KEY `tpid_rpl` (`tpid`,`tag`),
+  UNIQUE KEY `tpid_rplid_destrates_timings_weight` (`tpid`,`tag`,`destrates_tag`,`timing_tag`)
+);
+
+--
+-- Table structure for table `tp_rate_profiles`
+--
+
+DROP TABLE IF EXISTS `tp_rating_profiles`;
+CREATE TABLE `tp_rating_profiles` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `loadid` varchar(64) NOT NULL,
+  `tenant` varchar(64) NOT NULL,
+  `category` varchar(32) NOT NULL,
+  `subject` varchar(64) NOT NULL,
+  `activation_time` varchar(26) NOT NULL,
+  `rating_plan_tag` varchar(64) NOT NULL,
+  `fallback_subjects` varchar(64),
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`id`),
+   KEY `tpid` (`tpid`),
+  KEY `tpid_loadid` (`tpid`, `loadid`),
+  UNIQUE KEY `tpid_loadid_tenant_category_subj_atime` (`tpid`,`loadid`, `tenant`,`category`,`subject`,`activation_time`)
+);
+
+--
+-- Table structure for table `tp_shared_groups`
+--
+
+DROP TABLE IF EXISTS `tp_shared_groups`;
+CREATE TABLE `tp_shared_groups` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tag` varchar(64) NOT NULL,
+  `account` varchar(64) NOT NULL,
+  `strategy` varchar(24) NOT NULL,
+  `rating_subject` varchar(24) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_shared_group` (`tpid`,`tag`,`account`,`strategy`,`rating_subject`)
+);
+
+--
+-- Table structure for table `tp_actions`
+--
+
+DROP TABLE IF EXISTS `tp_actions`;
+CREATE TABLE `tp_actions` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tag` varchar(64) NOT NULL,
+  `action` varchar(24) NOT NULL,
+  `extra_parameters` varchar(256) NOT NULL,
+  `filters` varchar(256) NOT NULL,
+  `balance_tag` varchar(64) NOT NULL,
+  `balance_type` varchar(24) NOT NULL,
+  `categories` varchar(32) NOT NULL,
+  `destination_tags` varchar(64) NOT NULL,
+  `rating_subject` varchar(64) NOT NULL,
+  `shared_groups` varchar(64) NOT NULL,
+  `expiry_time` varchar(26) NOT NULL,
+  `timing_tags` varchar(128) NOT NULL,
+  `units` varchar(256) NOT NULL,
+  `balance_weight` varchar(10) NOT NULL,
+  `balance_blocker` varchar(5) NOT NULL,
+  `balance_disabled` varchar(24) NOT NULL,
+  `weight` DECIMAL(8,2) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_action` (`tpid`,`tag`,`action`,`balance_tag`,`balance_type`,`expiry_time`,`timing_tags`,`destination_tags`,`shared_groups`,`balance_weight`,`weight`)
+);
+
+--
+-- Table structure for table `tp_action_timings`
+--
+
+DROP TABLE IF EXISTS `tp_action_plans`;
+CREATE TABLE `tp_action_plans` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tag` varchar(64) NOT NULL,
+  `actions_tag` varchar(64) NOT NULL,
+  `timing_tag` varchar(64) NOT NULL,
+  `weight` DECIMAL(8,2) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_action_schedule` (`tpid`,`tag`,`actions_tag`,`timing_tag`)
+);
+
+--
+-- Table structure for table `tp_action_triggers`
+--
+
+DROP TABLE IF EXISTS `tp_action_triggers`;
+CREATE TABLE `tp_action_triggers` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tag` varchar(64) NOT NULL,
+  `unique_id` varchar(64) NOT NULL,
+  `threshold_type` char(64) NOT NULL,
+  `threshold_value` DECIMAL(20,4) NOT NULL,
+  `recurrent` BOOLEAN NOT NULL,
+  `min_sleep` varchar(16) NOT NULL,
+  `expiry_time` varchar(26) NOT NULL,
+  `activation_time` varchar(26) NOT NULL,
+  `balance_tag` varchar(64) NOT NULL,
+  `balance_type` varchar(24) NOT NULL,
+  `balance_categories` varchar(32) NOT NULL,
+  `balance_destination_tags` varchar(64) NOT NULL,
+  `balance_rating_subject` varchar(64) NOT NULL,
+  `balance_shared_groups` varchar(64) NOT NULL,
+  `balance_expiry_time` varchar(26) NOT NULL,
+  `balance_timing_tags` varchar(128) NOT NULL,
+  `balance_weight` varchar(10) NOT NULL,
+  `balance_blocker` varchar(5) NOT NULL,
+  `balance_disabled` varchar(5) NOT NULL,
+  `actions_tag` varchar(64) NOT NULL,
+  `weight` DECIMAL(8,2) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_trigger_definition` (`tpid`,`tag`,`balance_tag`,`balance_type`,`threshold_type`,`threshold_value`,`balance_destination_tags`,`actions_tag`)
+);
+
+--
+-- Table structure for table `tp_account_actions`
+--
+
+DROP TABLE IF EXISTS `tp_account_actions`;
+CREATE TABLE `tp_account_actions` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `loadid` varchar(64) NOT NULL,
+  `tenant` varchar(64) NOT NULL,
+  `account` varchar(64) NOT NULL,
+  `action_plan_tag` varchar(64),
+  `action_triggers_tag` varchar(64),
+  `allow_negative` BOOLEAN NOT NULL,
+  `disabled` BOOLEAN NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_tp_account` (`tpid`,`loadid`,`tenant`,`account`)
+);
+
+--
+-- Table structure for table `tp_resources`
+--
+
+DROP TABLE IF EXISTS tp_resources;
+CREATE TABLE tp_resources (
+  `pk` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tenant` varchar(64) NOT NULL,
+  `id` varchar(64) NOT NULL,
+  `filter_ids` varchar(64) NOT NULL,
+  `activation_interval` varchar(64) NOT NULL,
+  `usage_ttl` varchar(32) NOT NULL,
+  `limit` varchar(64) NOT NULL,
+  `allocation_message` varchar(64) NOT NULL,
+  `blocker` BOOLEAN NOT NULL,
+  `stored` BOOLEAN NOT NULL,
+  `weight` decimal(8,2) NOT NULL,
+  `threshold_ids` varchar(64) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`pk`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_tp_resource` (`tpid`,`tenant`, `id`,`filter_ids` )
+);
+
+--
+-- Table structure for table `tp_stats`
+--
+
+DROP TABLE IF EXISTS tp_stats;
+CREATE TABLE tp_stats (
+  `pk` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tenant` varchar(64) NOT NULL,
+  `id` varchar(64) NOT NULL,
+  `filter_ids` varchar(64) NOT NULL,
+  `activation_interval` varchar(64) NOT NULL,
+  `queue_length` int(11) NOT NULL,
+  `ttl` varchar(32) NOT NULL,
+  `min_items` int(11) NOT NULL,
+  `metric_ids` varchar(128) NOT NULL,
+  `metric_filter_ids` varchar(64) NOT NULL,
+  `stored` BOOLEAN NOT NULL,
+  `blocker` BOOLEAN NOT NULL,
+  `weight` decimal(8,2) NOT NULL,
+  `threshold_ids` varchar(64) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`pk`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_tp_stats` (`tpid`,  `tenant`, `id`, `filter_ids`,`metric_ids`)
+);
+
+--
+-- Table structure for table `tp_threshold_cfgs`
+--
+
+DROP TABLE IF EXISTS tp_thresholds;
+CREATE TABLE tp_thresholds (
+  `pk` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tenant` varchar(64) NOT NULL,
+  `id` varchar(64) NOT NULL,
+  `filter_ids` varchar(64) NOT NULL,
+  `activation_interval` varchar(64) NOT NULL,
+  `max_hits` int(11) NOT NULL,
+  `min_hits` int(11) NOT NULL,
+  `min_sleep` varchar(16) NOT NULL,
+  `blocker` BOOLEAN NOT NULL,
+  `weight` decimal(8,2) NOT NULL,
+  `action_ids` varchar(64) NOT NULL,
+  `async` BOOLEAN NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`pk`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_tp_thresholds` (`tpid`,`tenant`, `id`,`filter_ids`,`action_ids`)
+);
+
+--
+-- Table structure for table `tp_filter`
+--
+
+DROP TABLE IF EXISTS tp_filters;
+CREATE TABLE tp_filters (
+  `pk` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tenant` varchar(64) NOT NULL,
+  `id` varchar(64) NOT NULL,
+  `type` varchar(16) NOT NULL,
+  `element` varchar(64) NOT NULL,
+  `values` varchar(256) NOT NULL,
+  `activation_interval` varchar(64) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`pk`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_tp_filters` (`tpid`,`tenant`, `id`, `type`, `element`)
+);
+
+--
+-- Table structure for table `tp_routes`
+--
+
+
+DROP TABLE IF EXISTS tp_routes;
+CREATE TABLE tp_routes (
+  `pk` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tenant` varchar(64) NOT NULL,
+  `id` varchar(64) NOT NULL,
+  `filter_ids` varchar(64) NOT NULL,
+  `activation_interval` varchar(64) NOT NULL,
+  `sorting` varchar(32) NOT NULL,
+  `sorting_parameters` varchar(64) NOT NULL,
+  `route_id` varchar(32) NOT NULL,
+  `route_filter_ids` varchar(64) NOT NULL,
+  `route_account_ids` varchar(64) NOT NULL,
+  `route_ratingplan_ids` varchar(64) NOT NULL,
+  `route_rate_profile_ids` varchar(64) NOT NULL,
+  `route_resource_ids` varchar(64) NOT NULL,
+  `route_stat_ids` varchar(64) NOT NULL,
+  `route_weight` decimal(8,2) NOT NULL,
+  `route_blocker` BOOLEAN NOT NULL,
+  `route_parameters` varchar(64) NOT NULL,
+  `weight` decimal(8,2) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`pk`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_tp_routes` (`tpid`,`tenant`,
+    `id`,`filter_ids`,`route_id`,`route_filter_ids`,`route_account_ids`,
+    `route_ratingplan_ids`,`route_resource_ids`,`route_stat_ids` )
+);
+
+--
+-- Table structure for table `tp_attributes`
+--
+
+DROP TABLE IF EXISTS tp_attributes;
+CREATE TABLE tp_attributes (
+  `pk` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tenant` varchar(64) NOT NULL,
+  `id` varchar(64) NOT NULL,
+  `contexts` varchar(64) NOT NULL,
+  `filter_ids` varchar(64) NOT NULL,
+  `activation_interval` varchar(64) NOT NULL,
+  `attribute_filter_ids` varchar(64) NOT NULL,
+  `path` varchar(64) NOT NULL,
+  `type` varchar(64) NOT NULL,
+  `value` varchar(64) NOT NULL,
+  `blocker` BOOLEAN NOT NULL,
+  `weight` decimal(8,2) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`pk`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_tp_attributes` (`tpid`,`tenant`,
+    `id`,`filter_ids`,`path`,`value` )
+);
+
+--
+-- Table structure for table `tp_chargers`
+--
+
+DROP TABLE IF EXISTS tp_chargers;
+CREATE TABLE tp_chargers (
+  `pk` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tenant` varchar(64) NOT NULL,
+  `id` varchar(64) NOT NULL,
+  `filter_ids` varchar(64) NOT NULL,
+  `activation_interval` varchar(64) NOT NULL,
+  `run_id` varchar(64) NOT NULL,
+  `attribute_ids` varchar(64) NOT NULL,
+  `weight` decimal(8,2) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`pk`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_tp_chargers` (`tpid`,`tenant`,
+    `id`,`filter_ids`,`run_id`,`attribute_ids`)
+);
+
+--
+-- Table structure for table `tp_dispatchers`
+--
+
+DROP TABLE IF EXISTS tp_dispatcher_profiles;
+CREATE TABLE tp_dispatcher_profiles (
+  `pk` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tenant` varchar(64) NOT NULL,
+  `id` varchar(64) NOT NULL,
+  `subsystems` varchar(64) NOT NULL,
+  `filter_ids` varchar(64) NOT NULL,
+  `activation_interval` varchar(64) NOT NULL,
+  `strategy` varchar(64) NOT NULL,
+  `strategy_parameters` varchar(64) NOT NULL,
+  `conn_id` varchar(64) NOT NULL,
+  `conn_filter_ids` varchar(64) NOT NULL,
+  `conn_weight` decimal(8,2) NOT NULL,
+  `conn_blocker` BOOLEAN NOT NULL,
+  `conn_parameters` varchar(64) NOT NULL,
+  `weight` decimal(8,2) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`pk`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_tp_dispatcher_profiles` (`tpid`,`tenant`,
+    `id`,`filter_ids`,`strategy`,`conn_id`,`conn_filter_ids`)
+);
+
+--
+-- Table structure for table `tp_dispatchers`
+--
+
+DROP TABLE IF EXISTS tp_dispatcher_hosts;
+CREATE TABLE tp_dispatcher_hosts (
+  `pk` int(11) NOT NULL AUTO_INCREMENT,
+  `tpid` varchar(64) NOT NULL,
+  `tenant` varchar(64) NOT NULL,
+  `id` varchar(64) NOT NULL,
+  `address` varchar(64) NOT NULL,
+  `transport` varchar(64) NOT NULL,
+  `connect_attempts` int(11) NOT NULL,
+  `reconnects` int(11) NOT NULL,
+  `max_reconnect_interval` varchar(64) NOT NULL,
+  `connect_timeout` varchar(64) NOT NULL,
+  `reply_timeout` varchar(64) NOT NULL,
+  `tls` BOOLEAN NOT NULL,
+  `client_key` varchar(64) NOT NULL,
+  `client_certificate` varchar(64) NOT NULL,
+  `ca_certificate` varchar(64) NOT NULL,
+  `created_at` TIMESTAMP,
+  PRIMARY KEY (`pk`),
+  KEY `tpid` (`tpid`),
+  UNIQUE KEY `unique_tp_dispatchers_hosts` (`tpid`,`tenant`,
+    `id`,`address`)
+);
+
+--
+-- Table structure for table `versions`
+--
+
+DROP TABLE IF EXISTS versions;
+CREATE TABLE versions (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `item` varchar(64) NOT NULL,
+  `version` int(11) NOT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `id_item` (`id`,`item`)
+);

data/docker/integration/scripts/mysql/mysql_cdr_migration.sql

@@ -0,0 +1,57 @@
+/*
+This script will migrate CDRs from the old CGRateS tables to the new cdrs table
+but it only migrate CDRs where the duration is > 0.
+If you need CDRs also with duration is = 0 you can make the appropriate change in the line beginning WHERE cdrs_primary.usage
+Also the script will process 10,000 CDRs before committing to save system resources 
+especially in systems where they are millions of CDRs to be migrated
+You can increase or lower the value of step in the line after BEGIN below.
+
+You have to use 'CALL cgrates.migration();' to execute the script. If named other then default use that database name.
+*/
+
+
+DELIMITER //
+
+CREATE PROCEDURE `migration`()
+BEGIN
+        /* DECLARE variables */
+        DECLARE max_cdrs bigint;
+        DECLARE start_id bigint;
+        DECLARE end_id bigint;
+        DECLARE step bigint;
+        /* Optimize table for performance */
+        ALTER TABLE cdrs DISABLE KEYS;
+        SET autocommit=0;
+        SET unique_checks=0;
+        SET foreign_key_checks=0;
+        /* You must change the step var to commit every step rows inserted */
+        SET step := 10000;
+        SET start_id := 0;
+        SET end_id := start_id + step;
+        SET max_cdrs = (select max(id) from rated_cdrs);
+        WHILE (start_id <= max_cdrs) DO
+                INSERT INTO
+			cdrs(cgrid,run_id,origin_host,source,origin_id,tor,request_type,tenant,category,account,subject,destination,setup_time,pdd,answer_time,`usage`,supplier,disconnect_cause,extra_fields,cost_source,cost,cost_details,extra_info, created_at, updated_at, deleted_at) 
+			SELECT cdrs_primary.cgrid,rated_cdrs.runid as run_id,cdrs_primary.cdrhost as origin_host,cdrs_primary.cdrsource as source,cdrs_primary.accid as origin_id, cdrs_primary.tor,rated_cdrs.reqtype as request_type, rated_cdrs.tenant,rated_cdrs.category, rated_cdrs.account, rated_cdrs.subject, rated_cdrs.destination,rated_cdrs.setup_time,rated_cdrs.pdd,rated_cdrs.answer_time,rated_cdrs.`usage`,rated_cdrs.supplier,rated_cdrs.disconnect_cause,cdrs_extra.extra_fields,cost_details.cost_source,rated_cdrs.cost,cost_details.timespans as cost_details,rated_cdrs.extra_info,rated_cdrs.created_at,rated_cdrs.updated_at, rated_cdrs.deleted_at 
+                        FROM rated_cdrs
+                        INNER JOIN cdrs_primary ON rated_cdrs.cgrid = cdrs_primary.cgrid
+                        INNER JOIN cdrs_extra ON rated_cdrs.cgrid = cdrs_extra.cgrid
+                        INNER JOIN cost_details ON rated_cdrs.cgrid = cost_details.cgrid
+                        WHERE cdrs_primary.`usage` > '0'
+                     	AND not exists (select 1 from cdrs where cdrs.cgrid = cdrs_primary.cgrid AND cdrs.run_id=rated_cdrs.runid)
+						AND rated_cdrs.id >= start_id
+						AND rated_cdrs.id < end_id
+                    	GROUP BY cgrid, run_id, origin_id;
+                SET start_id = start_id + step;
+                SET end_id = end_id + step;
+        END WHILE;
+        /* SET Table for live usage */
+       SET autocommit=1;
+        SET unique_checks=1;
+        SET foreign_key_checks=1;
+        ALTER TABLE cdrs ENABLE KEYS;
+        OPTIMIZE TABLE cdrs;
+END //
+
+DELIMITER ;
+

data/docker/integration/scripts/postgres

@@ -1 +0,0 @@
-../../../storage/postgres/

data/docker/integration/scripts/postgres/create_cdrs_tables.sql

@@ -0,0 +1,59 @@
+--
+-- Table structure for table `cdrs`
+--
+
+DROP TABLE IF EXISTS cdrs;
+CREATE TABLE cdrs (
+ id SERIAL PRIMARY KEY,
+ cgrid VARCHAR(40) NOT NULL,
+ run_id VARCHAR(64) NOT NULL,
+ origin_host VARCHAR(64) NOT NULL,
+ source VARCHAR(64) NOT NULL,
+ origin_id VARCHAR(128) NOT NULL,
+ tor VARCHAR(16) NOT NULL,
+ request_type VARCHAR(24) NOT NULL,
+ tenant VARCHAR(64) NOT NULL,
+ category VARCHAR(64) NOT NULL,
+ account VARCHAR(128) NOT NULL,
+ subject VARCHAR(128) NOT NULL,
+ destination VARCHAR(128) NOT NULL,
+ setup_time TIMESTAMP WITH TIME ZONE NOT NULL,
+ answer_time TIMESTAMP WITH TIME ZONE NULL,
+ usage BIGINT NOT NULL,
+ extra_fields jsonb NOT NULL,
+ cost_source VARCHAR(64) NOT NULL,
+ cost NUMERIC(20,4) DEFAULT NULL,
+ cost_details jsonb,
+ extra_info text,
+ created_at TIMESTAMP WITH TIME ZONE,
+ updated_at TIMESTAMP WITH TIME ZONE NULL,
+ deleted_at TIMESTAMP WITH TIME ZONE NULL,
+ UNIQUE (cgrid, run_id)
+);
+;
+DROP INDEX IF EXISTS deleted_at_cp_idx;
+CREATE INDEX deleted_at_cp_idx ON cdrs (deleted_at);
+
+
+DROP TABLE IF EXISTS session_costs;
+CREATE TABLE session_costs (
+  id SERIAL PRIMARY KEY,
+  cgrid VARCHAR(40) NOT NULL,
+  run_id  VARCHAR(64) NOT NULL,
+  origin_host VARCHAR(64) NOT NULL,
+  origin_id VARCHAR(128) NOT NULL,
+  cost_source VARCHAR(64) NOT NULL,
+  usage BIGINT NOT NULL,
+  cost_details jsonb,
+  created_at TIMESTAMP WITH TIME ZONE,
+  deleted_at TIMESTAMP WITH TIME ZONE NULL,
+  UNIQUE (cgrid, run_id)
+);
+DROP INDEX IF EXISTS cgrid_sessionscost_idx;
+CREATE INDEX cgrid_sessionscost_idx ON session_costs (cgrid, run_id);
+DROP INDEX IF EXISTS origin_sessionscost_idx;
+CREATE INDEX origin_sessionscost_idx ON session_costs (origin_host, origin_id);
+DROP INDEX IF EXISTS run_origin_sessionscost_idx;
+CREATE INDEX run_origin_sessionscost_idx ON session_costs (run_id, origin_id);
+DROP INDEX IF EXISTS deleted_at_sessionscost_idx;
+CREATE INDEX deleted_at_sessionscost_idx ON session_costs (deleted_at);

data/docker/integration/scripts/postgres/create_tariffplan_tables.sql

@@ -0,0 +1,478 @@
+--
+-- Table structure for table `tp_timings`
+--
+DROP TABLE IF EXISTS tp_timings;
+CREATE TABLE tp_timings (
+  id SERIAL PRIMARY KEY,
+  tpid VARCHAR(64) NOT NULL,
+  tag VARCHAR(64) NOT NULL,
+  years VARCHAR(255) NOT NULL,
+  months VARCHAR(255) NOT NULL,
+  month_days VARCHAR(255) NOT NULL,
+  week_days VARCHAR(255) NOT NULL,
+  time VARCHAR(32) NOT NULL,
+  created_at TIMESTAMP WITH TIME ZONE,
+  UNIQUE  (tpid, tag)
+);
+CREATE INDEX tptimings_tpid_idx ON tp_timings (tpid);
+CREATE INDEX tptimings_idx ON tp_timings (tpid,tag);
+
+--
+-- Table structure for table `tp_destinations`
+--
+
+DROP TABLE IF EXISTS tp_destinations;
+CREATE TABLE tp_destinations (
+  id SERIAL PRIMARY KEY,
+  tpid VARCHAR(64) NOT NULL,
+  tag VARCHAR(64) NOT NULL,
+  prefix VARCHAR(24) NOT NULL,
+  created_at TIMESTAMP WITH TIME ZONE,
+  UNIQUE (tpid, tag, prefix)
+);
+CREATE INDEX tpdests_tpid_idx ON tp_destinations (tpid);
+CREATE INDEX tpdests_idx ON tp_destinations (tpid,tag);
+
+--
+-- Table structure for table `tp_rates`
+--
+
+DROP TABLE IF EXISTS tp_rates;
+CREATE TABLE tp_rates (
+  id SERIAL PRIMARY KEY,
+  tpid VARCHAR(64) NOT NULL,
+  tag VARCHAR(64) NOT NULL,
+  connect_fee NUMERIC(7,4) NOT NULL,
+  rate NUMERIC(10,4) NOT NULL,
+  rate_unit VARCHAR(16) NOT NULL,
+  rate_increment VARCHAR(16) NOT NULL,
+  group_interval_start VARCHAR(16) NOT NULL,
+  created_at TIMESTAMP WITH TIME ZONE,
+  UNIQUE (tpid, tag, group_interval_start)
+);
+CREATE INDEX tprates_tpid_idx ON tp_rates (tpid);
+CREATE INDEX tprates_idx ON tp_rates (tpid,tag);
+
+--
+-- Table structure for table `destination_rates`
+--
+
+DROP TABLE IF EXISTS tp_destination_rates;
+CREATE TABLE tp_destination_rates (
+  id SERIAL PRIMARY KEY,
+  tpid VARCHAR(64) NOT NULL,
+  tag VARCHAR(64) NOT NULL,
+  destinations_tag VARCHAR(64) NOT NULL,
+  rates_tag VARCHAR(64) NOT NULL,
+  rounding_method VARCHAR(255) NOT NULL,
+  rounding_decimals SMALLINT NOT NULL,
+  max_cost NUMERIC(7,4) NOT NULL,
+  max_cost_strategy VARCHAR(16) NOT NULL,
+  created_at TIMESTAMP WITH TIME ZONE,
+  UNIQUE (tpid, tag , destinations_tag)
+);
+CREATE INDEX tpdestrates_tpid_idx ON tp_destination_rates (tpid);
+CREATE INDEX tpdestrates_idx ON tp_destination_rates (tpid,tag);
+
+--
+-- Table structure for table `tp_rating_plans`
+--
+
+DROP TABLE IF EXISTS tp_rating_plans;
+CREATE TABLE tp_rating_plans (
+  id SERIAL PRIMARY KEY,
+  tpid VARCHAR(64) NOT NULL,
+  tag VARCHAR(64) NOT NULL,
+  destrates_tag VARCHAR(64) NOT NULL,
+  timing_tag VARCHAR(64) NOT NULL,
+  weight NUMERIC(8,2) NOT NULL,
+  created_at TIMESTAMP WITH TIME ZONE,
+  UNIQUE (tpid, tag, destrates_tag, timing_tag)
+);
+CREATE INDEX tpratingplans_tpid_idx ON tp_rating_plans (tpid);
+CREATE INDEX tpratingplans_idx ON tp_rating_plans (tpid,tag);
+
+
+--
+-- Table structure for table `tp_rate_profiles`
+--
+
+DROP TABLE IF EXISTS tp_rating_profiles;
+CREATE TABLE tp_rating_profiles (
+  id SERIAL PRIMARY KEY,
+  tpid VARCHAR(64) NOT NULL,
+  loadid VARCHAR(64) NOT NULL,
+  tenant VARCHAR(64) NOT NULL,
+  category VARCHAR(32) NOT NULL,
+  subject VARCHAR(64) NOT NULL,
+  activation_time VARCHAR(26) NOT NULL,
+  rating_plan_tag VARCHAR(64) NOT NULL,
+  fallback_subjects VARCHAR(64),
+  created_at TIMESTAMP WITH TIME ZONE,
+  UNIQUE (tpid, loadid, tenant, category, subject, activation_time)
+);
+CREATE INDEX tpratingprofiles_tpid_idx ON tp_rating_profiles (tpid);
+CREATE INDEX tpratingprofiles_idx ON tp_rating_profiles (tpid,loadid,tenant,category,subject);
+
+--
+-- Table structure for table `tp_shared_groups`
+--
+
+DROP TABLE IF EXISTS tp_shared_groups;
+CREATE TABLE tp_shared_groups (
+  id SERIAL PRIMARY KEY,
+  tpid VARCHAR(64) NOT NULL,
+  tag VARCHAR(64) NOT NULL,
+  account VARCHAR(64) NOT NULL,
+  strategy VARCHAR(24) NOT NULL,
+  rating_subject VARCHAR(24) NOT NULL,
+  created_at TIMESTAMP WITH TIME ZONE,
+  UNIQUE (tpid, tag, account , strategy , rating_subject)
+);
+CREATE INDEX tpsharedgroups_tpid_idx ON tp_shared_groups (tpid);
+CREATE INDEX tpsharedgroups_idx ON tp_shared_groups (tpid,tag);
+
+--
+-- Table structure for table `tp_actions`
+--
+
+DROP TABLE IF EXISTS tp_actions;
+CREATE TABLE tp_actions (
+  id SERIAL PRIMARY KEY,
+  tpid VARCHAR(64) NOT NULL,
+  tag VARCHAR(64) NOT NULL,
+  action VARCHAR(24) NOT NULL,
+  extra_parameters VARCHAR(256) NOT NULL,
+  filters VARCHAR(256) NOT NULL,
+  balance_tag VARCHAR(64) NOT NULL,
+  balance_type VARCHAR(24) NOT NULL,
+  categories VARCHAR(32) NOT NULL,
+  destination_tags VARCHAR(64) NOT NULL,
+  rating_subject VARCHAR(64) NOT NULL,
+  shared_groups VARCHAR(64) NOT NULL,
+  expiry_time VARCHAR(26) NOT NULL,
+  timing_tags VARCHAR(128) NOT NULL,
+  units VARCHAR(256) NOT NULL,
+  balance_weight VARCHAR(10) NOT NULL,
+  balance_blocker VARCHAR(5) NOT NULL,
+  balance_disabled VARCHAR(5) NOT NULL,
+  weight NUMERIC(8,2) NOT NULL,
+  created_at TIMESTAMP WITH TIME ZONE,
+  UNIQUE (tpid, tag, action, balance_tag, balance_type, expiry_time, timing_tags, destination_tags, shared_groups, balance_weight, weight)
+);
+CREATE INDEX tpactions_tpid_idx ON tp_actions (tpid);
+CREATE INDEX tpactions_idx ON tp_actions (tpid,tag);
+
+--
+-- Table structure for table `tp_action_timings`
+--
+
+DROP TABLE IF EXISTS tp_action_plans;
+CREATE TABLE tp_action_plans (
+  id SERIAL PRIMARY KEY,
+  tpid VARCHAR(64) NOT NULL,
+  tag VARCHAR(64) NOT NULL,
+  actions_tag VARCHAR(64) NOT NULL,
+  timing_tag VARCHAR(64) NOT NULL,
+  weight NUMERIC(8,2) NOT NULL,
+  created_at TIMESTAMP WITH TIME ZONE,
+  UNIQUE  (tpid, tag, actions_tag, timing_tag)
+);
+CREATE INDEX tpactionplans_tpid_idx ON tp_action_plans (tpid);
+CREATE INDEX tpactionplans_idx ON tp_action_plans (tpid,tag);
+
+--
+-- Table structure for table tp_action_triggers
+--
+
+DROP TABLE IF EXISTS tp_action_triggers;
+CREATE TABLE tp_action_triggers (
+  id SERIAL PRIMARY KEY,
+  tpid VARCHAR(64) NOT NULL,
+  tag VARCHAR(64) NOT NULL,
+  unique_id VARCHAR(64) NOT NULL,
+  threshold_type VARCHAR(64) NOT NULL,
+  threshold_value NUMERIC(20,4) NOT NULL,
+  recurrent BOOLEAN NOT NULL,
+  min_sleep VARCHAR(16) NOT NULL,
+  expiry_time VARCHAR(26) NOT NULL,
+  activation_time VARCHAR(26) NOT NULL,
+  balance_tag VARCHAR(64) NOT NULL,
+  balance_type VARCHAR(24) NOT NULL,
+  balance_categories VARCHAR(32) NOT NULL,
+  balance_destination_tags VARCHAR(64) NOT NULL,
+  balance_rating_subject VARCHAR(64) NOT NULL,
+  balance_shared_groups VARCHAR(64) NOT NULL,
+  balance_expiry_time VARCHAR(26) NOT NULL,
+  balance_timing_tags VARCHAR(128) NOT NULL,
+  balance_weight VARCHAR(10) NOT NULL,
+  balance_blocker VARCHAR(5) NOT NULL,
+  balance_disabled VARCHAR(5) NOT NULL,
+  actions_tag VARCHAR(64) NOT NULL,
+  weight NUMERIC(8,2) NOT NULL,
+  created_at TIMESTAMP WITH TIME ZONE,
+  UNIQUE (tpid, tag, balance_tag, balance_type, threshold_type, threshold_value, balance_destination_tags, actions_tag)
+);
+CREATE INDEX tpactiontrigers_tpid_idx ON tp_action_triggers (tpid);
+CREATE INDEX tpactiontrigers_idx ON tp_action_triggers (tpid,tag);
+
+--
+-- Table structure for table tp_account_actions
+--
+
+DROP TABLE IF EXISTS tp_account_actions;
+CREATE TABLE tp_account_actions (
+  id SERIAL PRIMARY KEY,
+  tpid VARCHAR(64) NOT NULL,
+  loadid VARCHAR(64) NOT NULL,
+  tenant VARCHAR(64) NOT NULL,
+  account VARCHAR(64) NOT NULL,
+  action_plan_tag VARCHAR(64),
+  action_triggers_tag VARCHAR(64),
+  allow_negative BOOLEAN NOT NULL,
+  disabled BOOLEAN NOT NULL,
+  created_at TIMESTAMP WITH TIME ZONE,
+  UNIQUE (tpid, loadid, tenant, account)
+);
+CREATE INDEX tpaccountactions_tpid_idx ON tp_account_actions (tpid);
+CREATE INDEX tpaccountactions_idx ON tp_account_actions (tpid,loadid,tenant,account);
+
+
+--
+-- Table structure for table `tp_resources`
+--
+
+DROP TABLE IF EXISTS tp_resources;
+CREATE TABLE tp_resources (
+  "pk" SERIAL PRIMARY KEY,
+  "tpid" varchar(64) NOT NULL,
+  "tenant"varchar(64) NOT NULL,
+  "id" varchar(64) NOT NULL,
+  "filter_ids" varchar(64) NOT NULL,
+  "activation_interval" varchar(64) NOT NULL,
+  "usage_ttl" varchar(32) NOT NULL,
+  "limit" varchar(64) NOT NULL,
+  "allocation_message" varchar(64) NOT NULL,
+  "blocker" BOOLEAN NOT NULL,
+  "stored" BOOLEAN NOT NULL,
+  "weight" NUMERIC(8,2) NOT NULL,
+  "threshold_ids" varchar(64) NOT NULL,
+  "created_at" TIMESTAMP WITH TIME ZONE
+);
+CREATE INDEX tp_resources_idx ON tp_resources (tpid);
+CREATE INDEX tp_resources_unique ON tp_resources  ("tpid",  "tenant", "id", "filter_ids");
+
+
+--
+-- Table structure for table `tp_stats`
+--
+
+DROP TABLE IF EXISTS tp_stats;
+CREATE TABLE tp_stats (
+  "pk" SERIAL PRIMARY KEY,
+  "tpid" varchar(64) NOT NULL,
+  "tenant"varchar(64) NOT NULL,
+  "id" varchar(64) NOT NULL,
+  "filter_ids" varchar(64) NOT NULL,
+  "activation_interval" varchar(64) NOT NULL,
+  "queue_length" INTEGER NOT NULL,
+  "ttl" varchar(32) NOT NULL,
+  "min_items" INTEGER NOT NULL,
+  "metric_ids" VARCHAR(128) NOT NULL,
+  "metric_filter_ids" VARCHAR(128) NOT NULL,
+  "stored" BOOLEAN NOT NULL,
+  "blocker" BOOLEAN NOT NULL,
+  "weight" decimal(8,2) NOT NULL,
+  "threshold_ids" varchar(64) NOT NULL,
+  "created_at" TIMESTAMP WITH TIME ZONE
+);
+CREATE INDEX tp_stats_idx ON tp_stats (tpid);
+CREATE INDEX tp_stats_unique ON tp_stats  ("tpid","tenant", "id", "filter_ids","metric_ids");
+
+--
+-- Table structure for table `tp_threshold_cfgs`
+--
+
+DROP TABLE IF EXISTS tp_thresholds;
+CREATE TABLE tp_thresholds (
+  "pk" SERIAL PRIMARY KEY,
+  "tpid" varchar(64) NOT NULL,
+  "tenant"varchar(64) NOT NULL,
+  "id" varchar(64) NOT NULL,
+  "filter_ids" varchar(64) NOT NULL,
+  "activation_interval" varchar(64) NOT NULL,
+  "max_hits" INTEGER NOT NULL,
+  "min_hits" INTEGER NOT NULL,
+  "min_sleep" varchar(16) NOT NULL,
+  "blocker" BOOLEAN NOT NULL,
+  "weight" decimal(8,2) NOT NULL,
+  "action_ids" varchar(64) NOT NULL,
+  "async" BOOLEAN NOT NULL,
+  "created_at" TIMESTAMP WITH TIME ZONE
+);
+CREATE INDEX tp_thresholds_idx ON tp_thresholds (tpid);
+CREATE INDEX tp_thresholds_unique ON tp_thresholds  ("tpid","tenant", "id","filter_ids","action_ids");
+
+--
+-- Table structure for table `tp_filter`
+--
+
+DROP TABLE IF EXISTS tp_filters;
+CREATE TABLE tp_filters (
+  "pk" SERIAL PRIMARY KEY,
+  "tpid" varchar(64) NOT NULL,
+  "tenant" varchar(64) NOT NULL,
+  "id" varchar(64) NOT NULL,
+  "type" varchar(16) NOT NULL,
+  "element" varchar(64) NOT NULL,
+  "values" varchar(256) NOT NULL,
+  "activation_interval" varchar(64) NOT NULL,
+  "created_at" TIMESTAMP WITH TIME ZONE
+);
+  CREATE INDEX tp_filters_idx ON tp_filters (tpid);
+  CREATE INDEX tp_filters_unique ON tp_filters  ("tpid","tenant", "id", "type", "element");
+
+--
+-- Table structure for table `tp_routes`
+--
+
+DROP TABLE IF EXISTS tp_routes;
+CREATE TABLE tp_routes (
+  "pk" SERIAL PRIMARY KEY,
+  "tpid" varchar(64) NOT NULL,
+  "tenant"varchar(64) NOT NULL,
+  "id" varchar(64) NOT NULL,
+  "filter_ids" varchar(64) NOT NULL,
+  "activation_interval" varchar(64) NOT NULL,
+  "sorting" varchar(32) NOT NULL,
+  "sorting_parameters" varchar(64) NOT NULL,
+  "route_id" varchar(32) NOT NULL,
+  "route_filter_ids" varchar(64) NOT NULL,
+  "route_account_ids" varchar(64) NOT NULL,
+  "route_ratingplan_ids" varchar(64) NOT NULL,
+  "route_resource_ids" varchar(64) NOT NULL,
+  "route_stat_ids" varchar(64) NOT NULL,
+  "route_weight" decimal(8,2) NOT NULL,
+  "route_blocker" BOOLEAN NOT NULL,
+  "route_parameters" varchar(64) NOT NULL,
+  "weight" decimal(8,2) NOT NULL,
+  "created_at" TIMESTAMP WITH TIME ZONE
+);
+CREATE INDEX tp_routes_idx ON tp_routes (tpid);
+CREATE INDEX tp_routes_unique ON tp_routes  ("tpid",  "tenant", "id",
+  "filter_ids","route_id","route_filter_ids","route_account_ids",
+  "route_ratingplan_ids","route_resource_ids","route_stat_ids");
+
+  --
+  -- Table structure for table `tp_attributes`
+  --
+
+  DROP TABLE IF EXISTS tp_attributes;
+  CREATE TABLE tp_attributes (
+    "pk" SERIAL PRIMARY KEY,
+    "tpid" varchar(64) NOT NULL,
+    "tenant"varchar(64) NOT NULL,
+    "id" varchar(64) NOT NULL,
+    "contexts" varchar(64) NOT NULL,
+    "filter_ids" varchar(64) NOT NULL,
+    "activation_interval" varchar(64) NOT NULL,
+    "attribute_filter_ids" varchar(64) NOT NULL,
+    "path" varchar(64) NOT NULL,
+    "type" varchar(64) NOT NULL,
+    "value" varchar(64) NOT NULL,
+    "blocker" BOOLEAN NOT NULL,
+    "weight" decimal(8,2) NOT NULL,
+    "created_at" TIMESTAMP WITH TIME ZONE
+  );
+  CREATE INDEX tp_attributes_ids ON tp_attributes (tpid);
+  CREATE INDEX tp_attributes_unique ON tp_attributes  ("tpid",  "tenant", "id",
+    "filter_ids","path","value");
+
+  --
+  -- Table structure for table `tp_chargers`
+  --
+
+  DROP TABLE IF EXISTS tp_chargers;
+  CREATE TABLE tp_chargers (
+    "pk" SERIAL PRIMARY KEY,
+    "tpid" varchar(64) NOT NULL,
+    "tenant"varchar(64) NOT NULL,
+    "id" varchar(64) NOT NULL,
+    "filter_ids" varchar(64) NOT NULL,
+    "activation_interval" varchar(64) NOT NULL,
+    "run_id" varchar(64) NOT NULL,
+    "attribute_ids" varchar(64) NOT NULL,
+    "weight" decimal(8,2) NOT NULL,
+    "created_at" TIMESTAMP WITH TIME ZONE
+  );
+  CREATE INDEX tp_chargers_ids ON tp_chargers (tpid);
+  CREATE INDEX tp_chargers_unique ON tp_chargers  ("tpid",  "tenant", "id",
+    "filter_ids","run_id","attribute_ids");
+
+  --
+  -- Table structure for table `tp_dispatchers`
+  --
+
+  DROP TABLE IF EXISTS tp_dispatcher_profiles;
+  CREATE TABLE tp_dispatcher_profiles (
+  "pk" SERIAL PRIMARY KEY,
+  "tpid" varchar(64) NOT NULL,
+  "tenant" varchar(64) NOT NULL,
+  "id" varchar(64) NOT NULL,
+  "subsystems" varchar(64) NOT NULL,
+  "filter_ids" varchar(64) NOT NULL,
+  "activation_interval" varchar(64) NOT NULL,
+  "strategy" varchar(64) NOT NULL,
+  "strategy_parameters" varchar(64) NOT NULL,
+  "conn_id" varchar(64) NOT NULL,
+  "conn_filter_ids" varchar(64) NOT NULL,
+  "conn_weight" decimal(8,2) NOT NULL,
+  "conn_blocker" BOOLEAN NOT NULL,
+  "conn_parameters" varchar(64) NOT NULL,
+  "weight" decimal(8,2) NOT NULL,
+  "created_at" TIMESTAMP WITH TIME ZONE
+  );
+  CREATE INDEX tp_dispatcher_profiles_ids ON tp_dispatcher_profiles (tpid);
+  CREATE INDEX tp_dispatcher_profiles_unique ON tp_dispatcher_profiles  ("tpid",  "tenant", "id",
+    "filter_ids","strategy","conn_id","conn_filter_ids");
+
+--
+-- Table structure for table `tp_dispatchers`
+--
+
+  DROP TABLE IF EXISTS tp_dispatcher_hosts;
+  CREATE TABLE tp_dispatcher_hosts (
+  "pk" SERIAL PRIMARY KEY,
+  "tpid" varchar(64) NOT NULL,
+  "tenant" varchar(64) NOT NULL,
+  "id" varchar(64) NOT NULL,
+  "address" varchar(64) NOT NULL,
+  "transport" varchar(64) NOT NULL,
+  "connect_attempts" INTEGER NOT NULL,
+  "reconnects" INTEGER NOT NULL,
+  "max_reconnect_interval" varchar(64) NOT NULL,
+  "connect_timeout" varchar(64) NOT NULL,
+  "reply_timeout" varchar(64) NOT NULL,
+  "tls" BOOLEAN NOT NULL,
+  "client_key" varchar(64) NOT NULL,
+  "client_certificate" varchar(64) NOT NULL,
+  "ca_certificate" varchar(64) NOT NULL,
+  "created_at" TIMESTAMP WITH TIME ZONE
+  );
+  CREATE INDEX tp_dispatchers_hosts_ids ON tp_dispatcher_hosts (tpid);
+  CREATE INDEX tp_dispatcher_hosts_unique ON tp_dispatcher_hosts  ("tpid",  "tenant", "id",
+    "address");
+
+
+
+--
+-- Table structure for table `versions`
+--
+
+DROP TABLE IF EXISTS versions;
+CREATE TABLE versions (
+  "id" SERIAL PRIMARY KEY,
+  "item" varchar(64) NOT NULL,
+  "version" INTEGER NOT NULL,
+  UNIQUE ("id","item")
+);

data/docker/integration/scripts/postgres/pg_cdr_migration.sql

@@ -0,0 +1,42 @@
+/*
+This script will migrate CDRs from the old CGRateS tables to the new cdrs table
+but it only migrate CDRs where the duration is > 0.
+If you need CDRs also with duration is = 0 you can make the appropriate change in the line beginning WHERE cdrs_primary.usage
+
+Also the script will process 10,000 CDRs before committing to save system resources 
+especially in systems where they are millions of CDRs to be migrated
+You can increase or lower the value of step in the line after BEGIN below.
+*/
+
+DO $$
+DECLARE
+    max_cdrs bigint;
+    start_id bigint;
+    end_id bigint;
+    step bigint;
+BEGIN
+	/* You must change the step var to commit every step rows inserted */
+	step := 10000;
+	start_id := 0;
+	end_id := start_id + step;
+	select max(id) INTO max_cdrs from rated_cdrs;
+	WHILE start_id <= max_cdrs
+	LOOP
+		--RAISE NOTICE '%', (to_char(start_id, '99999999') || '-' || to_char(end_id, '99999999'));
+		INSERT INTO 
+			cdrs(cgrid,run_id,origin_host,source,origin_id,tor,request_type,tenant,category,account,subject,destination,setup_time,pdd,answer_time,usage,supplier,disconnect_cause,extra_fields,cost_source,cost,cost_details,extra_info, created_at, updated_at, deleted_at) 
+			SELECT cdrs_primary.cgrid,rated_cdrs.runid as run_id,cdrs_primary.cdrhost as origin_host,cdrs_primary.cdrsource as source,cdrs_primary.accid as origin_id, cdrs_primary.tor,rated_cdrs.reqtype as request_type, rated_cdrs.tenant,rated_cdrs.category, rated_cdrs.account, rated_cdrs.subject, rated_cdrs.destination,rated_cdrs.setup_time,rated_cdrs.pdd,rated_cdrs.answer_time,rated_cdrs.usage,rated_cdrs.supplier,rated_cdrs.disconnect_cause,cdrs_extra.extra_fields,cost_details.cost_source,rated_cdrs.cost,cost_details.timespans as cost_details,rated_cdrs.extra_info,rated_cdrs.created_at,rated_cdrs.updated_at, rated_cdrs.deleted_at 
+			FROM rated_cdrs 
+			INNER JOIN cdrs_primary ON rated_cdrs.cgrid = cdrs_primary.cgrid 
+			INNER JOIN cdrs_extra ON rated_cdrs.cgrid = cdrs_extra.cgrid 
+			INNER JOIN cost_details ON rated_cdrs.cgrid = cost_details.cgrid 
+			WHERE cdrs_primary.usage > '0'
+			AND not exists (select 1 from cdrs c where c.cgrid = cdrs_primary.cgrid)
+			AND rated_cdrs.id >= start_id
+			AND rated_cdrs.id < end_id
+			;
+		start_id = start_id + step;
+		end_id = end_id + step;
+	END LOOP;
+END 
+$$;